2024-09-17T10:29:32.443 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-17T10:29:32.461 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T10:29:32.548 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690 branch: main description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7907690' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 7498493279982dfe87f00616198a5967475f6169 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 7498493279982dfe87f00616198a5967475f6169 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 2b698643d05f73e79c2d2888228947a84a4ac0bd owner: scheduled_teuthology@teuthology priority: 950 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 3649 sha1: 7498493279982dfe87f00616198a5967475f6169 sleep_before_teardown: 0 subset: 42/64 suite: orch suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_2b698643d05f73e79c2d2888228947a84a4ac0bd/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 2b698643d05f73e79c2d2888228947a84a4ac0bd targets: smithi006.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK97Bk8OYOSVsl2pmiR+mhhLEsL4dxhrZdl4ZZSJMnY4h8rVC9hvLAFS+3F46LbzlK1N9vc5JelF60eiRWWJfVY= smithi149.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG3gT8KKsSm+lwtkfPs3hY4u4njHGi6S4CeZuGOd/JowHWOcjIaXMVYctyJklesUEoV3aOn9VqYdieVowKdFwHE= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-16_20:08:15 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-17T10:29:32.548 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_2b698643d05f73e79c2d2888228947a84a4ac0bd/qa; will attempt to use it 2024-09-17T10:29:32.549 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_2b698643d05f73e79c2d2888228947a84a4ac0bd/qa/tasks 2024-09-17T10:29:32.549 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-17T10:29:32.551 INFO:teuthology.task.internal:Checking packages... 2024-09-17T10:29:32.573 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '7498493279982dfe87f00616198a5967475f6169' 2024-09-17T10:29:32.574 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-17T10:29:32.574 INFO:teuthology.packaging:ref: None 2024-09-17T10:29:32.574 INFO:teuthology.packaging:tag: None 2024-09-17T10:29:32.574 INFO:teuthology.packaging:branch: main 2024-09-17T10:29:32.574 INFO:teuthology.packaging:sha1: 7498493279982dfe87f00616198a5967475f6169 2024-09-17T10:29:32.574 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-17T10:29:32.793 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4959.g2b698643 2024-09-17T10:29:32.794 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-17T10:29:32.802 INFO:teuthology.task.internal:no buildpackages task found 2024-09-17T10:29:32.802 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-17T10:29:32.821 INFO:teuthology.task.internal:Saving configuration 2024-09-17T10:29:32.835 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-17T10:29:32.842 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-17T10:29:32.865 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi006.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 10:24:32.680236', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK97Bk8OYOSVsl2pmiR+mhhLEsL4dxhrZdl4ZZSJMnY4h8rVC9hvLAFS+3F46LbzlK1N9vc5JelF60eiRWWJfVY='} 2024-09-17T10:29:32.884 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi149.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 10:24:32.678454', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG3gT8KKsSm+lwtkfPs3hY4u4njHGi6S4CeZuGOd/JowHWOcjIaXMVYctyJklesUEoV3aOn9VqYdieVowKdFwHE='} 2024-09-17T10:29:32.884 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-17T10:29:32.890 INFO:teuthology.task.internal:roles: ubuntu@smithi006.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-17T10:29:32.890 INFO:teuthology.task.internal:roles: ubuntu@smithi149.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-17T10:29:32.890 INFO:teuthology.run_tasks:Running task console_log... 2024-09-17T10:29:32.946 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f0e5f8f7ac0>, signals=[15]) 2024-09-17T10:29:32.946 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-17T10:29:32.953 INFO:teuthology.task.internal:Opening connections... 2024-09-17T10:29:32.953 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:29:32.954 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:29:33.034 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi149.front.sepia.ceph.com 2024-09-17T10:29:33.035 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:29:33.107 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-17T10:29:33.114 DEBUG:teuthology.orchestra.run.smithi006:> uname -m 2024-09-17T10:29:33.132 INFO:teuthology.orchestra.run.smithi006.stdout:x86_64 2024-09-17T10:29:33.132 DEBUG:teuthology.orchestra.run.smithi006:> cat /etc/os-release 2024-09-17T10:29:33.190 INFO:teuthology.orchestra.run.smithi006.stdout:NAME="CentOS Stream" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:VERSION="9" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:ID="centos" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:ID_LIKE="rhel fedora" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:VERSION_ID="9" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:PLATFORM_ID="platform:el9" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:ANSI_COLOR="0;31" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:LOGO="fedora-logo-icon" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:HOME_URL="https://centos.org/" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T10:29:33.191 INFO:teuthology.orchestra.run.smithi006.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T10:29:33.192 INFO:teuthology.orchestra.run.smithi006.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T10:29:33.192 INFO:teuthology.lock.ops:Updating smithi006.front.sepia.ceph.com on lock server 2024-09-17T10:29:33.222 DEBUG:teuthology.orchestra.run.smithi149:> uname -m 2024-09-17T10:29:33.240 INFO:teuthology.orchestra.run.smithi149.stdout:x86_64 2024-09-17T10:29:33.240 DEBUG:teuthology.orchestra.run.smithi149:> cat /etc/os-release 2024-09-17T10:29:33.298 INFO:teuthology.orchestra.run.smithi149.stdout:NAME="CentOS Stream" 2024-09-17T10:29:33.298 INFO:teuthology.orchestra.run.smithi149.stdout:VERSION="9" 2024-09-17T10:29:33.298 INFO:teuthology.orchestra.run.smithi149.stdout:ID="centos" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:ID_LIKE="rhel fedora" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:VERSION_ID="9" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:PLATFORM_ID="platform:el9" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:ANSI_COLOR="0;31" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:LOGO="fedora-logo-icon" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:HOME_URL="https://centos.org/" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T10:29:33.299 INFO:teuthology.orchestra.run.smithi149.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T10:29:33.300 INFO:teuthology.lock.ops:Updating smithi149.front.sepia.ceph.com on lock server 2024-09-17T10:29:33.322 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-17T10:29:33.332 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-17T10:29:33.338 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-17T10:29:33.339 DEBUG:teuthology.orchestra.run.smithi006:> test '!' -e /home/ubuntu/cephtest 2024-09-17T10:29:33.342 DEBUG:teuthology.orchestra.run.smithi149:> test '!' -e /home/ubuntu/cephtest 2024-09-17T10:29:33.360 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-17T10:29:33.365 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-17T10:29:33.366 DEBUG:teuthology.orchestra.run.smithi006:> test -z $(ls -A /var/lib/ceph) 2024-09-17T10:29:33.399 DEBUG:teuthology.orchestra.run.smithi149:> test -z $(ls -A /var/lib/ceph) 2024-09-17T10:29:33.502 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-17T10:29:33.565 INFO:teuthology.run_tasks:Running task kernel... 2024-09-17T10:29:33.581 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-17T10:29:33.582 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-17T10:29:33.582 DEBUG:teuthology.orchestra.run.smithi006:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T10:29:33.582 DEBUG:teuthology.orchestra.run.smithi149:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T10:29:33.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T10:29:33.599 DEBUG:teuthology.orchestra.run.smithi006:> uname -r 2024-09-17T10:29:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T10:29:33.600 DEBUG:teuthology.orchestra.run.smithi149:> uname -r 2024-09-17T10:29:33.657 INFO:teuthology.orchestra.run.smithi006.stdout:5.14.0-503.el9.x86_64 2024-09-17T10:29:33.657 INFO:teuthology.task.kernel:Running kernel on smithi006: 5.14.0-503.el9.x86_64 2024-09-17T10:29:33.657 DEBUG:teuthology.orchestra.run.smithi006:> sudo yum install -y kernel 2024-09-17T10:29:33.658 INFO:teuthology.orchestra.run.smithi149.stdout:5.14.0-503.el9.x86_64 2024-09-17T10:29:33.658 INFO:teuthology.task.kernel:Running kernel on smithi149: 5.14.0-503.el9.x86_64 2024-09-17T10:29:33.659 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum install -y kernel 2024-09-17T10:29:35.946 INFO:teuthology.orchestra.run.smithi006.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.3 MB 00:00 2024-09-17T10:29:36.184 INFO:teuthology.orchestra.run.smithi149.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.3 MB 00:00 2024-09-17T10:29:39.139 INFO:teuthology.orchestra.run.smithi006.stdout:CentOS Stream 9 - AppStream 9.9 MB/s | 20 MB 00:02 2024-09-17T10:29:43.490 INFO:teuthology.orchestra.run.smithi006.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-17T10:29:45.290 INFO:teuthology.orchestra.run.smithi006.stdout:CentOS Stream 9 - Extras packages 34 kB/s | 19 kB 00:00 2024-09-17T10:29:46.161 INFO:teuthology.orchestra.run.smithi006.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-17T10:29:51.717 INFO:teuthology.orchestra.run.smithi006.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-17T10:29:53.707 INFO:teuthology.orchestra.run.smithi006.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T10:29:53.707 INFO:teuthology.orchestra.run.smithi006.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T10:29:53.789 INFO:teuthology.orchestra.run.smithi006.stdout:Dependencies resolved. 2024-09-17T10:29:53.795 INFO:teuthology.orchestra.run.smithi006.stdout:================================================================================ 2024-09-17T10:29:53.795 INFO:teuthology.orchestra.run.smithi006.stdout: Package Architecture Version Repository Size 2024-09-17T10:29:53.795 INFO:teuthology.orchestra.run.smithi006.stdout:================================================================================ 2024-09-17T10:29:53.795 INFO:teuthology.orchestra.run.smithi006.stdout:Installing: 2024-09-17T10:29:53.795 INFO:teuthology.orchestra.run.smithi006.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T10:29:53.795 INFO:teuthology.orchestra.run.smithi006.stdout:Installing dependencies: 2024-09-17T10:29:53.795 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T10:29:53.795 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T10:29:53.796 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T10:29:53.796 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:29:53.796 INFO:teuthology.orchestra.run.smithi006.stdout:Transaction Summary 2024-09-17T10:29:53.796 INFO:teuthology.orchestra.run.smithi006.stdout:================================================================================ 2024-09-17T10:29:53.796 INFO:teuthology.orchestra.run.smithi006.stdout:Install 4 Packages 2024-09-17T10:29:53.796 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:29:53.796 INFO:teuthology.orchestra.run.smithi006.stdout:Total download size: 79 M 2024-09-17T10:29:53.797 INFO:teuthology.orchestra.run.smithi006.stdout:Installed size: 126 M 2024-09-17T10:29:53.797 INFO:teuthology.orchestra.run.smithi006.stdout:Downloading Packages: 2024-09-17T10:29:53.893 INFO:teuthology.orchestra.run.smithi006.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 470 kB/s | 36 kB 00:00 2024-09-17T10:29:56.649 INFO:teuthology.orchestra.run.smithi149.stdout:CentOS Stream 9 - AppStream 1.1 MB/s | 20 MB 00:19 2024-09-17T10:29:57.702 INFO:teuthology.orchestra.run.smithi006.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 4.0 MB/s | 16 MB 00:03 2024-09-17T10:30:01.243 INFO:teuthology.orchestra.run.smithi006.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 3.9 MB/s | 29 MB 00:07 2024-09-17T10:30:02.050 INFO:teuthology.orchestra.run.smithi149.stdout:CentOS Stream 9 - CRB 4.8 MB/s | 6.5 MB 00:01 2024-09-17T10:30:02.735 INFO:teuthology.orchestra.run.smithi006.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 3.9 MB/s | 35 MB 00:08 2024-09-17T10:30:02.735 INFO:teuthology.orchestra.run.smithi006.stdout:-------------------------------------------------------------------------------- 2024-09-17T10:30:02.735 INFO:teuthology.orchestra.run.smithi006.stdout:Total 8.8 MB/s | 79 MB 00:08 2024-09-17T10:30:03.444 INFO:teuthology.orchestra.run.smithi006.stdout:Running transaction check 2024-09-17T10:30:03.825 INFO:teuthology.orchestra.run.smithi006.stdout:Transaction check succeeded. 2024-09-17T10:30:03.825 INFO:teuthology.orchestra.run.smithi006.stdout:Running transaction test 2024-09-17T10:30:03.925 INFO:teuthology.orchestra.run.smithi149.stdout:CentOS Stream 9 - Extras packages 31 kB/s | 19 kB 00:00 2024-09-17T10:30:04.805 INFO:teuthology.orchestra.run.smithi149.stdout:Extra Packages for Enterprise Linux 30 MB/s | 23 MB 00:00 2024-09-17T10:30:04.832 INFO:teuthology.orchestra.run.smithi006.stdout:Transaction test succeeded. 2024-09-17T10:30:04.833 INFO:teuthology.orchestra.run.smithi006.stdout:Running transaction 2024-09-17T10:30:07.385 INFO:teuthology.orchestra.run.smithi006.stdout: Preparing : 1/1 2024-09-17T10:30:08.046 INFO:teuthology.orchestra.run.smithi006.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T10:30:08.136 INFO:teuthology.orchestra.run.smithi006.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T10:30:10.494 INFO:teuthology.orchestra.run.smithi149.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-17T10:30:10.906 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T10:30:11.032 INFO:teuthology.orchestra.run.smithi006.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T10:30:12.400 INFO:teuthology.orchestra.run.smithi149.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T10:30:12.400 INFO:teuthology.orchestra.run.smithi149.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T10:30:12.492 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2024-09-17T10:30:12.497 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout: Package Architecture Version Repository Size 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout:Install 4 Packages 2024-09-17T10:30:12.498 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:30:12.499 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 79 M 2024-09-17T10:30:12.499 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 126 M 2024-09-17T10:30:12.499 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2024-09-17T10:30:12.753 INFO:teuthology.orchestra.run.smithi149.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 248 kB/s | 36 kB 00:00 2024-09-17T10:30:13.237 INFO:teuthology.orchestra.run.smithi149.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 25 MB/s | 16 MB 00:00 2024-09-17T10:30:13.646 INFO:teuthology.orchestra.run.smithi149.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 32 MB/s | 29 MB 00:00 2024-09-17T10:30:13.954 INFO:teuthology.orchestra.run.smithi149.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 26 MB/s | 35 MB 00:01 2024-09-17T10:30:13.955 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2024-09-17T10:30:13.955 INFO:teuthology.orchestra.run.smithi149.stdout:Total 54 MB/s | 79 MB 00:01 2024-09-17T10:30:14.633 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2024-09-17T10:30:14.998 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2024-09-17T10:30:14.998 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2024-09-17T10:30:15.936 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2024-09-17T10:30:15.937 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2024-09-17T10:30:16.148 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T10:30:16.191 INFO:teuthology.orchestra.run.smithi006.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:30:18.395 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2024-09-17T10:30:19.055 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T10:30:19.146 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T10:30:21.328 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:30:21.915 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T10:30:22.049 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T10:30:26.871 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T10:30:26.958 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:30:31.797 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:00.659 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:00.700 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:00.998 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:00.998 INFO:teuthology.orchestra.run.smithi006.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T10:31:00.998 INFO:teuthology.orchestra.run.smithi006.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T10:31:00.998 INFO:teuthology.orchestra.run.smithi006.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T10:31:01.891 INFO:teuthology.orchestra.run.smithi006.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:01.891 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:31:01.891 INFO:teuthology.orchestra.run.smithi006.stdout:Installed: 2024-09-17T10:31:01.892 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T10:31:01.892 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T10:31:01.892 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T10:31:01.892 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T10:31:01.892 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:31:01.892 INFO:teuthology.orchestra.run.smithi006.stdout:Complete! 2024-09-17T10:31:02.244 DEBUG:teuthology.orchestra.run.smithi006:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T10:31:02.329 INFO:teuthology.orchestra.run.smithi006.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T10:31:02.330 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T10:31:02.330 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T10:31:02.330 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T10:31:02.330 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T10:31:02.330 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-17T10:31:02.330 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi006.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T10:31:02.331 DEBUG:teuthology.orchestra.run.smithi006:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T10:31:02.389 INFO:teuthology.orchestra.run.smithi006.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T10:31:02.390 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T10:31:02.390 DEBUG:teuthology.orchestra.run.smithi006:> sudo rpm -qi grub2-tools 2024-09-17T10:31:02.421 INFO:teuthology.orchestra.run.smithi006.stdout:Name : grub2-tools 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Epoch : 1 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Version : 2.06 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Release : 82.el9 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Architecture: x86_64 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Group : Unspecified 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Size : 8274670 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:License : GPLv3+ 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T10:31:02.422 INFO:teuthology.orchestra.run.smithi006.stdout:Packager : builder@centos.org 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:Vendor : CentOS 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:Summary : Support tools for GRUB. 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:Description : 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:hardware devices. 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:31:02.423 INFO:teuthology.orchestra.run.smithi006.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T10:31:02.424 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T10:31:02.424 INFO:teuthology.task.kernel:Updating grub on smithi006 to boot 5.14.0-508.el9.x86_64 2024-09-17T10:31:02.424 DEBUG:teuthology.orchestra.run.smithi006:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T10:31:03.081 INFO:teuthology.orchestra.run.smithi006.stderr:Generating grub configuration file ... 2024-09-17T10:31:04.532 INFO:teuthology.orchestra.run.smithi006.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T10:31:04.560 INFO:teuthology.orchestra.run.smithi006.stderr:done 2024-09-17T10:31:04.562 DEBUG:teuthology.orchestra.run.smithi006:> mktemp 2024-09-17T10:31:04.598 INFO:teuthology.orchestra.run.smithi006.stdout:/tmp/tmp.lz4JPY7mN6 2024-09-17T10:31:04.598 DEBUG:teuthology.orchestra.run.smithi006:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.lz4JPY7mN6 2024-09-17T10:31:04.666 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 0666 /tmp/tmp.lz4JPY7mN6 2024-09-17T10:31:04.813 DEBUG:teuthology.orchestra.remote:smithi006:/tmp/tmp.lz4JPY7mN6 is 6KB 2024-09-17T10:31:04.834 DEBUG:teuthology.orchestra.run.smithi006:> rm -fr /tmp/tmp.lz4JPY7mN6 2024-09-17T10:31:04.849 DEBUG:teuthology.orchestra.run.smithi006:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T10:31:04.911 INFO:teuthology.orchestra.run.smithi006.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T10:31:04.911 INFO:teuthology.orchestra.run.smithi006.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T10:31:04.911 INFO:teuthology.orchestra.run.smithi006.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T10:31:04.911 INFO:teuthology.orchestra.run.smithi006.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T10:31:04.913 DEBUG:teuthology.orchestra.run.smithi006:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T10:31:05.094 DEBUG:teuthology.orchestra.run.smithi006:> sudo shutdown -r now 2024-09-17T10:31:10.045 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:10.084 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:10.366 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:10.366 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T10:31:10.366 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T10:31:10.366 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T10:31:11.205 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T10:31:11.206 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:31:11.206 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2024-09-17T10:31:11.206 INFO:teuthology.orchestra.run.smithi149.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T10:31:11.206 INFO:teuthology.orchestra.run.smithi149.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T10:31:11.206 INFO:teuthology.orchestra.run.smithi149.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T10:31:11.206 INFO:teuthology.orchestra.run.smithi149.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T10:31:11.207 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:31:11.207 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2024-09-17T10:31:11.605 DEBUG:teuthology.orchestra.run.smithi149:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T10:31:11.685 INFO:teuthology.orchestra.run.smithi149.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T10:31:11.685 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T10:31:11.685 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T10:31:11.685 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T10:31:11.686 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T10:31:11.686 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-17T10:31:11.686 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi149.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T10:31:11.686 DEBUG:teuthology.orchestra.run.smithi149:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T10:31:11.745 INFO:teuthology.orchestra.run.smithi149.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T10:31:11.746 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T10:31:11.746 DEBUG:teuthology.orchestra.run.smithi149:> sudo rpm -qi grub2-tools 2024-09-17T10:31:11.782 INFO:teuthology.orchestra.run.smithi149.stdout:Name : grub2-tools 2024-09-17T10:31:11.782 INFO:teuthology.orchestra.run.smithi149.stdout:Epoch : 1 2024-09-17T10:31:11.782 INFO:teuthology.orchestra.run.smithi149.stdout:Version : 2.06 2024-09-17T10:31:11.782 INFO:teuthology.orchestra.run.smithi149.stdout:Release : 82.el9 2024-09-17T10:31:11.782 INFO:teuthology.orchestra.run.smithi149.stdout:Architecture: x86_64 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Group : Unspecified 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Size : 8274670 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:License : GPLv3+ 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Packager : builder@centos.org 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Vendor : CentOS 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Summary : Support tools for GRUB. 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:Description : 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:31:11.783 INFO:teuthology.orchestra.run.smithi149.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T10:31:11.784 INFO:teuthology.orchestra.run.smithi149.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T10:31:11.784 INFO:teuthology.orchestra.run.smithi149.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T10:31:11.784 INFO:teuthology.orchestra.run.smithi149.stdout:hardware devices. 2024-09-17T10:31:11.784 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:31:11.784 INFO:teuthology.orchestra.run.smithi149.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T10:31:11.785 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T10:31:11.785 INFO:teuthology.task.kernel:Updating grub on smithi149 to boot 5.14.0-508.el9.x86_64 2024-09-17T10:31:11.785 DEBUG:teuthology.orchestra.run.smithi149:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T10:31:12.483 INFO:teuthology.orchestra.run.smithi149.stderr:Generating grub configuration file ... 2024-09-17T10:31:13.953 INFO:teuthology.orchestra.run.smithi149.stderr:File descriptor 3 (pipe:[38593]) leaked on vgs invocation. Parent PID 22674: grub2-probe 2024-09-17T10:31:13.953 INFO:teuthology.orchestra.run.smithi149.stderr:File descriptor 9 (pipe:[51631]) leaked on vgs invocation. Parent PID 22674: grub2-probe 2024-09-17T10:31:13.988 INFO:teuthology.orchestra.run.smithi149.stderr:File descriptor 3 (pipe:[38593]) leaked on vgs invocation. Parent PID 22674: grub2-probe 2024-09-17T10:31:13.988 INFO:teuthology.orchestra.run.smithi149.stderr:File descriptor 9 (pipe:[51631]) leaked on vgs invocation. Parent PID 22674: grub2-probe 2024-09-17T10:31:14.155 INFO:teuthology.orchestra.run.smithi149.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T10:31:14.173 INFO:teuthology.orchestra.run.smithi149.stderr:done 2024-09-17T10:31:14.176 DEBUG:teuthology.orchestra.run.smithi149:> mktemp 2024-09-17T10:31:14.191 INFO:teuthology.orchestra.run.smithi149.stdout:/tmp/tmp.EaWhO1vu1P 2024-09-17T10:31:14.192 DEBUG:teuthology.orchestra.run.smithi149:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.EaWhO1vu1P 2024-09-17T10:31:14.259 DEBUG:teuthology.orchestra.run.smithi149:> sudo chmod 0666 /tmp/tmp.EaWhO1vu1P 2024-09-17T10:31:14.400 DEBUG:teuthology.orchestra.remote:smithi149:/tmp/tmp.EaWhO1vu1P is 6KB 2024-09-17T10:31:14.449 DEBUG:teuthology.orchestra.run.smithi149:> rm -fr /tmp/tmp.EaWhO1vu1P 2024-09-17T10:31:14.466 DEBUG:teuthology.orchestra.run.smithi149:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T10:31:14.528 INFO:teuthology.orchestra.run.smithi149.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T10:31:14.528 INFO:teuthology.orchestra.run.smithi149.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T10:31:14.528 INFO:teuthology.orchestra.run.smithi149.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T10:31:14.529 INFO:teuthology.orchestra.run.smithi149.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T10:31:14.530 DEBUG:teuthology.orchestra.run.smithi149:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T10:31:14.692 DEBUG:teuthology.orchestra.run.smithi149:> sudo shutdown -r now 2024-09-17T10:31:35.117 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-17T10:31:35.117 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-09-17T10:31:35.118 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:31:44.703 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-17T10:31:44.704 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-09-17T10:31:44.704 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:31:53.665 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.6 2024-09-17T10:32:02.675 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-09-17T10:32:02.675 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:32:03.137 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.149 2024-09-17T10:32:12.146 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-09-17T10:32:12.146 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:32:15.197 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.149 2024-09-17T10:32:27.209 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-09-17T10:32:27.209 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:33:02.710 DEBUG:teuthology.orchestra.remote:timed out 2024-09-17T10:33:14.721 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-09-17T10:33:14.722 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:33:15.023 DEBUG:teuthology.orchestra.run.smithi006:> true 2024-09-17T10:33:15.403 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-09-17T10:33:15.403 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-17T10:33:15.403 DEBUG:teuthology.orchestra.run.smithi006:> uname -r 2024-09-17T10:33:15.420 INFO:teuthology.orchestra.run.smithi006.stdout:5.14.0-508.el9.x86_64 2024-09-17T10:33:15.420 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T10:33:15.420 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T10:33:15.421 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T10:33:16.422 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-17T10:33:16.422 DEBUG:teuthology.orchestra.run.smithi006:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T10:33:16.521 INFO:teuthology.orchestra.run.smithi006.stdout:ttyS1 2024-09-17T10:33:16.541 DEBUG:teuthology.parallel:result is None 2024-09-17T10:33:27.211 DEBUG:teuthology.orchestra.remote:timed out 2024-09-17T10:33:42.212 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-09-17T10:33:42.213 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:33:42.504 DEBUG:teuthology.orchestra.run.smithi149:> true 2024-09-17T10:33:42.871 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-09-17T10:33:42.871 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-17T10:33:42.872 DEBUG:teuthology.orchestra.run.smithi149:> uname -r 2024-09-17T10:33:42.887 INFO:teuthology.orchestra.run.smithi149.stdout:5.14.0-508.el9.x86_64 2024-09-17T10:33:42.887 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T10:33:42.887 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T10:33:42.887 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T10:33:43.888 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-17T10:33:43.888 DEBUG:teuthology.orchestra.run.smithi149:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T10:33:43.973 INFO:teuthology.orchestra.run.smithi149.stdout:ttyS1 2024-09-17T10:33:43.999 DEBUG:teuthology.parallel:result is None 2024-09-17T10:33:43.999 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-17T10:33:44.006 INFO:teuthology.task.internal:Creating test directory... 2024-09-17T10:33:44.007 DEBUG:teuthology.orchestra.run.smithi006:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T10:33:44.009 DEBUG:teuthology.orchestra.run.smithi149:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T10:33:44.027 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-17T10:33:44.036 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-17T10:33:44.194 INFO:teuthology.task.internal:Creating archive directory... 2024-09-17T10:33:44.194 DEBUG:teuthology.orchestra.run.smithi006:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T10:33:44.197 DEBUG:teuthology.orchestra.run.smithi149:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T10:33:44.229 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-17T10:33:44.235 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-17T10:33:44.235 DEBUG:teuthology.orchestra.run.smithi006:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T10:33:44.261 DEBUG:teuthology.orchestra.run.smithi149:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T10:33:44.297 INFO:teuthology.orchestra.run.smithi006.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T10:33:44.300 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T10:33:44.309 INFO:teuthology.orchestra.run.smithi006.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T10:33:44.312 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T10:33:44.314 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-17T10:33:44.320 INFO:teuthology.task.internal:Configuring sudo... 2024-09-17T10:33:44.320 DEBUG:teuthology.orchestra.run.smithi006:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T10:33:44.353 DEBUG:teuthology.orchestra.run.smithi149:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T10:33:44.381 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-17T10:33:44.390 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-17T10:33:44.390 DEBUG:teuthology.orchestra.run.smithi006:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T10:33:44.422 DEBUG:teuthology.orchestra.run.smithi149:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T10:33:44.438 DEBUG:teuthology.orchestra.run.smithi006:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T10:33:44.522 DEBUG:teuthology.orchestra.run.smithi006:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T10:33:44.568 DEBUG:teuthology.orchestra.run.smithi006:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T10:33:44.652 DEBUG:teuthology.orchestra.run.smithi006:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T10:33:44.695 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:33:44.695 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T10:33:44.763 DEBUG:teuthology.orchestra.run.smithi149:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T10:33:44.804 DEBUG:teuthology.orchestra.run.smithi149:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T10:33:44.866 DEBUG:teuthology.orchestra.run.smithi149:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T10:33:44.953 DEBUG:teuthology.orchestra.run.smithi149:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T10:33:44.995 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-09-17T10:33:44.995 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T10:33:45.063 DEBUG:teuthology.orchestra.run.smithi006:> sudo service rsyslog restart 2024-09-17T10:33:45.065 DEBUG:teuthology.orchestra.run.smithi149:> sudo service rsyslog restart 2024-09-17T10:33:45.115 INFO:teuthology.orchestra.run.smithi006.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T10:33:45.150 INFO:teuthology.orchestra.run.smithi149.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T10:33:45.491 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-17T10:33:45.498 INFO:teuthology.task.internal:Starting timer... 2024-09-17T10:33:45.498 INFO:teuthology.run_tasks:Running task pcp... 2024-09-17T10:33:45.508 INFO:teuthology.run_tasks:Running task selinux... 2024-09-17T10:33:45.518 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-17T10:33:45.518 DEBUG:teuthology.orchestra.run.smithi006:> sudo service auditd rotate 2024-09-17T10:33:45.576 INFO:teuthology.orchestra.run.smithi006.stdout:Rotating logs: 2024-09-17T10:33:45.577 DEBUG:teuthology.orchestra.run.smithi149:> sudo service auditd rotate 2024-09-17T10:33:45.627 INFO:teuthology.orchestra.run.smithi149.stdout:Rotating logs: 2024-09-17T10:33:45.629 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-17T10:33:45.629 DEBUG:teuthology.orchestra.run.smithi006:> /usr/sbin/getenforce 2024-09-17T10:33:45.655 INFO:teuthology.orchestra.run.smithi006.stdout:Permissive 2024-09-17T10:33:45.655 DEBUG:teuthology.orchestra.run.smithi149:> /usr/sbin/getenforce 2024-09-17T10:33:45.678 INFO:teuthology.orchestra.run.smithi149.stdout:Permissive 2024-09-17T10:33:45.678 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi006.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi149.front.sepia.ceph.com': 'permissive'} 2024-09-17T10:33:45.678 DEBUG:teuthology.orchestra.run.smithi006:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T10:33:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T10:33:45.724 DEBUG:teuthology.orchestra.run.smithi149:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T10:33:45.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T10:33:45.752 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-17T10:33:45.753 DEBUG:teuthology.orchestra.run.smithi006:> sudo /usr/sbin/setenforce permissive 2024-09-17T10:33:45.810 DEBUG:teuthology.orchestra.run.smithi149:> sudo /usr/sbin/setenforce permissive 2024-09-17T10:33:45.846 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-17T10:33:45.855 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-17T10:33:45.867 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-17T10:33:45.868 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi006.front.sepia.ceph.com,smithi149.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-17T10:37:41.116 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi006.front.sepia.ceph.com'), Remote(name='ubuntu@smithi149.front.sepia.ceph.com')] 2024-09-17T10:37:41.117 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-09-17T10:37:41.118 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:37:41.194 DEBUG:teuthology.orchestra.run.smithi006:> true 2024-09-17T10:37:41.272 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-09-17T10:37:41.273 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-09-17T10:37:41.273 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T10:37:41.345 DEBUG:teuthology.orchestra.run.smithi149:> true 2024-09-17T10:37:41.423 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-09-17T10:37:41.424 INFO:teuthology.run_tasks:Running task clock... 2024-09-17T10:37:41.434 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-17T10:37:41.434 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T10:37:41.435 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T10:37:41.437 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T10:37:41.437 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T10:37:41.464 INFO:teuthology.orchestra.run.smithi006.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T10:37:41.479 INFO:teuthology.orchestra.run.smithi006.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T10:37:41.492 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T10:37:41.505 INFO:teuthology.orchestra.run.smithi006.stderr:sudo: ntpd: command not found 2024-09-17T10:37:41.506 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T10:37:41.517 INFO:teuthology.orchestra.run.smithi006.stdout:506 Cannot talk to daemon 2024-09-17T10:37:41.530 INFO:teuthology.orchestra.run.smithi149.stderr:sudo: ntpd: command not found 2024-09-17T10:37:41.531 INFO:teuthology.orchestra.run.smithi006.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T10:37:41.541 INFO:teuthology.orchestra.run.smithi149.stdout:506 Cannot talk to daemon 2024-09-17T10:37:41.546 INFO:teuthology.orchestra.run.smithi006.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T10:37:41.556 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T10:37:41.570 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T10:37:41.602 INFO:teuthology.orchestra.run.smithi006.stderr:bash: line 1: ntpq: command not found 2024-09-17T10:37:41.605 INFO:teuthology.orchestra.run.smithi006.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T10:37:41.605 INFO:teuthology.orchestra.run.smithi006.stdout:=============================================================================== 2024-09-17T10:37:41.605 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T10:37:41.605 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T10:37:41.605 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T10:37:41.605 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T10:37:41.625 INFO:teuthology.orchestra.run.smithi149.stderr:bash: line 1: ntpq: command not found 2024-09-17T10:37:41.628 INFO:teuthology.orchestra.run.smithi149.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T10:37:41.628 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================== 2024-09-17T10:37:41.629 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T10:37:41.629 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T10:37:41.629 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T10:37:41.629 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T10:37:41.630 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-17T10:37:41.723 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '7498493279982dfe87f00616198a5967475f6169'} 2024-09-17T10:37:41.723 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-17T10:37:41.724 INFO:tasks.cephadm:Cluster fsid is ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:37:41.724 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-17T10:37:41.724 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-17T10:37:41.724 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi006': '172.21.15.6', 'mon.smithi149': '172.21.15.149'} 2024-09-17T10:37:41.724 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-17T10:37:41.724 DEBUG:teuthology.orchestra.run.smithi006:> sudo hostname $(hostname -s) 2024-09-17T10:37:41.751 DEBUG:teuthology.orchestra.run.smithi149:> sudo hostname $(hostname -s) 2024-09-17T10:37:41.779 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-17T10:37:41.779 DEBUG:teuthology.orchestra.run.smithi006:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T10:37:42.083 INFO:teuthology.orchestra.run.smithi006.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 10:37 /home/ubuntu/cephtest/cephadm 2024-09-17T10:37:42.084 DEBUG:teuthology.orchestra.run.smithi149:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T10:37:42.201 INFO:teuthology.orchestra.run.smithi149.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 10:37 /home/ubuntu/cephtest/cephadm 2024-09-17T10:37:42.202 DEBUG:teuthology.orchestra.run.smithi006:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T10:37:42.220 DEBUG:teuthology.orchestra.run.smithi149:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T10:37:42.244 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-17T10:37:42.244 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T10:37:42.262 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T10:37:43.828 INFO:teuthology.orchestra.run.smithi006.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T10:37:43.884 INFO:teuthology.orchestra.run.smithi149.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T10:37:56.400 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2024-09-17T10:37:56.401 INFO:teuthology.orchestra.run.smithi149.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T10:37:56.401 INFO:teuthology.orchestra.run.smithi149.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T10:37:56.401 INFO:teuthology.orchestra.run.smithi149.stdout: "repo_digests": [ 2024-09-17T10:37:56.401 INFO:teuthology.orchestra.run.smithi149.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T10:37:56.401 INFO:teuthology.orchestra.run.smithi149.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T10:37:56.401 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2024-09-17T10:37:56.401 INFO:teuthology.orchestra.run.smithi149.stdout:} 2024-09-17T10:37:56.423 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:37:56.423 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T10:37:56.423 INFO:teuthology.orchestra.run.smithi006.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T10:37:56.423 INFO:teuthology.orchestra.run.smithi006.stdout: "repo_digests": [ 2024-09-17T10:37:56.423 INFO:teuthology.orchestra.run.smithi006.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T10:37:56.423 INFO:teuthology.orchestra.run.smithi006.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T10:37:56.423 INFO:teuthology.orchestra.run.smithi006.stdout: ] 2024-09-17T10:37:56.423 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:37:56.444 DEBUG:teuthology.orchestra.run.smithi006:> sudo mkdir -p /etc/ceph 2024-09-17T10:37:56.500 DEBUG:teuthology.orchestra.run.smithi149:> sudo mkdir -p /etc/ceph 2024-09-17T10:37:56.529 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 777 /etc/ceph 2024-09-17T10:37:56.555 DEBUG:teuthology.orchestra.run.smithi149:> sudo chmod 777 /etc/ceph 2024-09-17T10:37:56.595 INFO:tasks.cephadm:Writing seed config... 2024-09-17T10:37:56.597 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-17T10:37:56.597 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-17T10:37:56.597 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-17T10:37:56.597 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-17T10:37:56.597 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-17T10:37:56.598 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-17T10:37:56.598 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-17T10:37:56.598 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-17T10:37:56.598 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:37:56.599 DEBUG:teuthology.orchestra.run.smithi006:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-17T10:37:56.615 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ddf74116-74e0-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-17T10:37:56.615 DEBUG:teuthology.orchestra.run.smithi006:mon.smithi006> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006.service 2024-09-17T10:37:56.657 INFO:tasks.cephadm:Bootstrapping... 2024-09-17T10:37:56.658 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid ddf74116-74e0-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.6 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-17T10:37:56.855 INFO:teuthology.orchestra.run.smithi006.stderr:-------------------------------------------------------------------------------- 2024-09-17T10:37:56.855 INFO:teuthology.orchestra.run.smithi006.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'ddf74116-74e0-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.6', '--skip-admin-label'] 2024-09-17T10:37:56.877 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: 5.2.2 2024-09-17T10:37:56.879 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying podman|docker is present... 2024-09-17T10:37:56.900 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: 5.2.2 2024-09-17T10:37:56.902 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying lvm2 is present... 2024-09-17T10:37:56.902 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying time synchronization is in place... 2024-09-17T10:37:56.909 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T10:37:56.916 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-09-17T10:37:56.923 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: enabled 2024-09-17T10:37:56.929 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: active 2024-09-17T10:37:56.930 INFO:teuthology.orchestra.run.smithi006.stderr:Unit chronyd.service is enabled and running 2024-09-17T10:37:56.930 INFO:teuthology.orchestra.run.smithi006.stderr:Repeating the final host check... 2024-09-17T10:37:56.950 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: 5.2.2 2024-09-17T10:37:56.953 INFO:teuthology.orchestra.run.smithi006.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-17T10:37:56.953 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl is present 2024-09-17T10:37:56.953 INFO:teuthology.orchestra.run.smithi006.stderr:lvcreate is present 2024-09-17T10:37:56.959 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T10:37:56.966 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-09-17T10:37:56.973 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: enabled 2024-09-17T10:37:56.979 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: active 2024-09-17T10:37:56.980 INFO:teuthology.orchestra.run.smithi006.stderr:Unit chronyd.service is enabled and running 2024-09-17T10:37:56.980 INFO:teuthology.orchestra.run.smithi006.stderr:Host looks OK 2024-09-17T10:37:56.980 INFO:teuthology.orchestra.run.smithi006.stderr:Cluster fsid: ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:37:56.980 INFO:teuthology.orchestra.run.smithi006.stderr:Acquiring lock 139988394616720 on /run/cephadm/ddf74116-74e0-11ef-bceb-c7b262605968.lock 2024-09-17T10:37:56.980 INFO:teuthology.orchestra.run.smithi006.stderr:Lock 139988394616720 acquired on /run/cephadm/ddf74116-74e0-11ef-bceb-c7b262605968.lock 2024-09-17T10:37:56.981 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying IP 172.21.15.6 port 3300 ... 2024-09-17T10:37:56.981 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying IP 172.21.15.6 port 6789 ... 2024-09-17T10:37:56.981 INFO:teuthology.orchestra.run.smithi006.stderr:Base mon IP is 172.21.15.6, final addrv is [v2:172.21.15.6:3300,v1:172.21.15.6:6789] 2024-09-17T10:37:56.983 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.6 metric 100 2024-09-17T10:37:56.983 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.6 metric 100 2024-09-17T10:37:56.986 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-17T10:37:56.986 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-17T10:37:56.986 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1787sec hoplimit 64 pref medium 2024-09-17T10:37:56.989 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-17T10:37:56.989 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-17T10:37:56.989 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T10:37:56.989 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-17T10:37:56.989 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:15e4/64 scope link 2024-09-17T10:37:56.989 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T10:37:56.990 INFO:teuthology.orchestra.run.smithi006.stderr:Mon IP `172.21.15.6` is in CIDR network `172.21.0.0/20` 2024-09-17T10:37:56.990 INFO:teuthology.orchestra.run.smithi006.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-17T10:37:56.990 INFO:teuthology.orchestra.run.smithi006.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T10:37:57.093 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-17T10:37:57.509 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Getting image source signatures 2024-09-17T10:37:57.509 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-17T10:37:57.510 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-17T10:37:57.510 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-17T10:37:57.510 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-17T10:37:57.510 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-17T10:37:57.510 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T10:37:57.643 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Writing manifest to image destination 2024-09-17T10:37:57.754 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T10:37:58.155 INFO:teuthology.orchestra.run.smithi006.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T10:37:58.704 INFO:teuthology.orchestra.run.smithi006.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T10:37:58.704 INFO:teuthology.orchestra.run.smithi006.stderr:Extracting ceph user uid/gid from container image... 2024-09-17T10:37:59.094 INFO:teuthology.orchestra.run.smithi006.stderr:stat: 167 167 2024-09-17T10:37:59.792 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial keys... 2024-09-17T10:38:01.823 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQAJXOlmlz5PBhAAqTKsYkac/6UwgiVgZzRNqg== 2024-09-17T10:38:11.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQASXOlmK1IHDxAAaaKYQAC3xTtOaXzto20WHA== 2024-09-17T10:38:12.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQAUXOlmFLWsCRAAJpuYWGTVN0FhnCVg0o/O8w== 2024-09-17T10:38:12.653 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial monmap... 2024-09-17T10:38:13.369 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T10:38:13.369 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-17T10:38:13.369 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:38:13.369 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T10:38:13.967 INFO:teuthology.orchestra.run.smithi006.stderr:monmaptool for smithi006 [v2:172.21.15.6:3300,v1:172.21.15.6:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T10:38:13.967 INFO:teuthology.orchestra.run.smithi006.stderr:setting min_mon_release = octopus 2024-09-17T10:38:13.968 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: set fsid to ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:38:13.968 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T10:38:13.968 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:38:13.968 INFO:teuthology.orchestra.run.smithi006.stderr:Creating mon... 2024-09-17T10:38:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:create mon.smithi006 on 2024-09-17T10:38:15.585 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-17T10:38:15.746 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ddf74116-74e0-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-ddf74116-74e0-11ef-bceb-c7b262605968.target. 2024-09-17T10:38:15.747 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-ddf74116-74e0-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-ddf74116-74e0-11ef-bceb-c7b262605968.target. 2024-09-17T10:38:16.142 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to reset failed state of unit ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006.service: Unit ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006.service not loaded. 2024-09-17T10:38:16.150 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ddf74116-74e0-11ef-bceb-c7b262605968.target.wants/ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006.service → /etc/systemd/system/ceph-ddf74116-74e0-11ef-bceb-c7b262605968@.service. 2024-09-17T10:38:17.030 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: disabled 2024-09-17T10:38:17.037 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-09-17T10:38:17.038 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld.service is not enabled 2024-09-17T10:38:17.038 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T10:38:17.038 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mon to start... 2024-09-17T10:38:17.038 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mon... 2024-09-17T10:38:17.525 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: cluster: 2024-09-17T10:38:17.525 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: id: ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:38:17.525 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-17T10:38:17.525 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:17.525 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: services: 2024-09-17T10:38:17.525 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi006 (age 0.540548s) 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: data: 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: pgs: 2024-09-17T10:38:17.526 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:18.106 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:17 smithi006 ceph-mon[23400]: from='client.? 172.21.15.6:0/4036611886' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T10:38:18.109 INFO:teuthology.orchestra.run.smithi006.stderr:mon is available 2024-09-17T10:38:18.109 INFO:teuthology.orchestra.run.smithi006.stderr:Assimilating anything we can from ceph.conf... 2024-09-17T10:38:18.651 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [global] 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: fsid = ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.6:3300,v1:172.21.15.6:6789] 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [mgr] 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:18.652 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [osd] 2024-09-17T10:38:18.653 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T10:38:18.653 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T10:38:19.187 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:18 smithi006 ceph-mon[23400]: from='client.? 172.21.15.6:0/1171918261' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T10:38:19.187 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:18 smithi006 ceph-mon[23400]: from='client.? 172.21.15.6:0/1171918261' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-17T10:38:19.190 INFO:teuthology.orchestra.run.smithi006.stderr:Generating new minimal ceph.conf... 2024-09-17T10:38:20.230 INFO:teuthology.orchestra.run.smithi006.stderr:Restarting the monitor... 2024-09-17T10:38:20.298 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:19 smithi006 ceph-mon[23400]: from='client.? 172.21.15.6:0/2727792170' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:38:20.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:20 smithi006 systemd[1]: Stopping Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:38:20.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:20 smithi006 bash[23619]: Error: no container with name or ID "ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon.smithi006" found: no such container 2024-09-17T10:38:20.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:20 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[23396]: 2024-09-17T10:38:20.387+0000 7fae3f529700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi006 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:38:20.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:20 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[23396]: 2024-09-17T10:38:20.387+0000 7fae3f529700 -1 mon.smithi006@0(leader) e1 *** Got Signal Terminated *** 2024-09-17T10:38:20.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:20 smithi006 podman[23631]: 2024-09-17 10:38:20.500411313 +0000 UTC m=+0.208583116 container died 90236f8c04c293365272f6664e0982e035b00d7a24d6dec99279ea2146098cbc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vcs-type=git, io.buildah.version=1.19.8, architecture=x86_64) 2024-09-17T10:38:21.031 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:20 smithi006 podman[23631]: 2024-09-17 10:38:20.837149364 +0000 UTC m=+0.545321166 container cleanup 90236f8c04c293365272f6664e0982e035b00d7a24d6dec99279ea2146098cbc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, name=centos-stream, architecture=x86_64, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_CLEAN=True, GIT_BRANCH=HEAD, version=8, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T10:38:21.031 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:20 smithi006 bash[23631]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006 2024-09-17T10:38:21.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 podman[23643]: 2024-09-17 10:38:21.029843199 +0000 UTC m=+0.524512598 container remove 90236f8c04c293365272f6664e0982e035b00d7a24d6dec99279ea2146098cbc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, architecture=x86_64, vcs-type=git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T10:38:21.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 bash[23658]: Error: no container with name or ID "ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon.smithi006" found: no such container 2024-09-17T10:38:21.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006.service: Deactivated successfully. 2024-09-17T10:38:21.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 systemd[1]: Stopped Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:38:21.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 systemd[1]: Starting Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:38:21.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 podman[23720]: 2024-09-17 10:38:21.272179148 +0000 UTC m=+0.103434122 container create 12038485a6a786222af5d3e206d29ae96d04b4fe184d1019539457e541beae3b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, ceph=True, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.openshift.tags=base centos centos-stream) 2024-09-17T10:38:21.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 podman[23720]: 2024-09-17 10:38:21.186194243 +0000 UTC m=+0.017449230 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-17T10:38:21.558 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 podman[23720]: 2024-09-17 10:38:21.490951419 +0000 UTC m=+0.322206409 container init 12038485a6a786222af5d3e206d29ae96d04b4fe184d1019539457e541beae3b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, build-date=2022-05-03T08:36:31.336870, release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, name=centos-stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, version=8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 podman[23720]: 2024-09-17 10:38:21.494752317 +0000 UTC m=+0.326007292 container start 12038485a6a786222af5d3e206d29ae96d04b4fe184d1019539457e541beae3b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vcs-type=git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, version=8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=) 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: pidfile_write: ignore empty --pid-file 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: load: jerasure load: lrc 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: RocksDB version: 6.15.5 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Compile date Apr 18 2022 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: DB SUMMARY 2024-09-17T10:38:21.559 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: DB Session ID: ZMJSN2T59NBF3RIKI06W 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: CURRENT file: CURRENT 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi006/store.db dir, Total Num: 1, files: 000008.sst 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi006/store.db: 000010.log size: 82687 ; 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.error_if_exists: 0 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.create_if_missing: 0 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.paranoid_checks: 1 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.env: 0x55a9c64c6860 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.fs: Posix File System 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.info_log: 0x55a9c7a31fa0 2024-09-17T10:38:21.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.statistics: (nil) 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.use_fsync: 0 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_log_file_size: 0 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.allow_fallocate: 1 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T10:38:21.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.use_direct_reads: 0 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.db_log_dir: 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi006/store.db 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T10:38:21.562 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T10:38:21.563 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T10:38:21.564 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T10:38:21.564 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.write_buffer_manager: 0x55a9c7b22330 2024-09-17T10:38:21.564 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T10:38:21.564 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T10:38:21.564 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T10:38:21.564 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T10:38:21.564 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T10:38:21.564 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.unordered_write: 0 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.row_cache: None 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.wal_filter: None 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.preserve_deletes: 0 2024-09-17T10:38:21.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.two_write_queues: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.atomic_flush: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.log_readahead_size: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T10:38:21.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_background_jobs: 2 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_background_compactions: -1 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_subcompactions: 1 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_open_files: -1 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T10:38:21.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_background_flushes: -1 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Compression algorithms supported: 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: kZSTD supported: 0 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: kXpressCompression supported: 0 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: kLZ4Compression supported: 1 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: kBZip2Compression supported: 0 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: kZlibCompression supported: 1 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: kSnappyCompression supported: 1 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T10:38:21.568 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi006/store.db/MANIFEST-000009 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.merge_operator: 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_filter: None 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a9c79fdd00) 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: cache_index_and_filter_blocks: 1 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: pin_top_level_index_and_filter: 1 2024-09-17T10:38:21.569 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_type: 0 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_block_index_type: 0 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_shortening: 1 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: hash_index_allow_collision: 1 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: checksum: 1 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: no_block_cache: 0 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache: 0x55a9c7a68170 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_name: BinnedLRUCache 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_options: 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: capacity : 536870912 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: num_shard_bits : 4 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: strict_capacity_limit : 0 2024-09-17T10:38:21.570 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: high_pri_pool_ratio: 0.000 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_compressed: (nil) 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: persistent_cache: (nil) 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_size: 4096 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_size_deviation: 10 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_restart_interval: 16 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_block_restart_interval: 1 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: metadata_block_size: 4096 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: partition_filters: 0 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: use_delta_encoding: 1 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: whole_key_filtering: 1 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: verify_compression: 0 2024-09-17T10:38:21.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: read_amp_bytes_per_bit: 0 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: format_version: 4 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: enable_index_compression: 1 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_align: 0 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compression: NoCompression 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.num_levels: 7 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T10:38:21.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T10:38:21.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T10:38:21.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.arena_block_size: 4194304 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T10:38:21.575 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.table_properties_collectors: 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.inplace_update_support: 0 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.bloom_locality: 0 2024-09-17T10:38:21.576 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.max_successive_merges: 0 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.ttl: 2592000 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.enable_blob_files: false 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.min_blob_size: 0 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T10:38:21.577 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi006/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726569501521757, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726569501522522, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79369, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77633, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71942, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726569501, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9a09a1d8-ace3-494c-a9e4-8ad9b832c557", "db_session_id": "ZMJSN2T59NBF3RIKI06W"}} 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726569501522855, "job": 1, "event": "recovery_finished"} 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi006/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55a9c7a4ea80 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: DB pointer 0x55a9c7a5e000 2024-09-17T10:38:21.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** DB Stats ** 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T10:38:21.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: L0 2/0 79.31 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Sum 2/0 79.31 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T10:38:21.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative compaction: 0.00 GB write, 24.19 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval compaction: 0.00 GB write, 24.19 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: L0 2/0 79.31 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Sum 2/0 79.31 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T10:38:21.581 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative compaction: 0.00 GB write, 23.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T10:38:21.582 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: starting mon.smithi006 rank 0 at public addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] at bind addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi006 fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006@-1(???) e1 preinit fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006@-1(???).mds e1 new map 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006@-1(???).mds e1 print_map 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: e1 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: legacy client fscid: -1 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: No filesystems configured 2024-09-17T10:38:21.583 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T10:38:21.584 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T10:38:21.584 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T10:38:21.584 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T10:38:21.584 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-17T10:38:21.584 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 bash[23720]: 12038485a6a786222af5d3e206d29ae96d04b4fe184d1019539457e541beae3b 2024-09-17T10:38:21.584 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 systemd[1]: Started Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:38:21.584 INFO:teuthology.orchestra.run.smithi006.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-17T10:38:21.862 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mon.smithi006 is new leader, mons smithi006 in quorum (ranks 0) 2024-09-17T10:38:21.862 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: monmap e1: 1 mons at {smithi006=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0]} 2024-09-17T10:38:21.862 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: fsmap 2024-09-17T10:38:21.862 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T10:38:21.862 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:21 smithi006 ceph-mon[23740]: mgrmap e1: no daemons active 2024-09-17T10:38:22.610 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-17T10:38:22.610 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-17T10:38:22.610 INFO:teuthology.orchestra.run.smithi006.stderr:Creating mgr... 2024-09-17T10:38:22.610 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying port 9283 ... 2024-09-17T10:38:22.797 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to reset failed state of unit ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mgr.smithi006.plhvdm.service: Unit ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mgr.smithi006.plhvdm.service not loaded. 2024-09-17T10:38:22.805 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ddf74116-74e0-11ef-bceb-c7b262605968.target.wants/ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mgr.smithi006.plhvdm.service → /etc/systemd/system/ceph-ddf74116-74e0-11ef-bceb-c7b262605968@.service. 2024-09-17T10:38:23.185 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:23 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2871246275' entity='client.admin' 2024-09-17T10:38:23.478 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: disabled 2024-09-17T10:38:23.485 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-09-17T10:38:23.486 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld.service is not enabled 2024-09-17T10:38:23.486 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T10:38:23.494 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: disabled 2024-09-17T10:38:23.502 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-09-17T10:38:23.503 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld.service is not enabled 2024-09-17T10:38:23.503 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-17T10:38:23.503 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr to start... 2024-09-17T10:38:23.503 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr... 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "ddf74116-74e0-11ef-bceb-c7b262605968", 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:24.416 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "smithi006" 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:24.417 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T10:38:24.418 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:24.418 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T10:38:24.418 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T10:38:24.418 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T10:38:24.418 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T10:38:24.421 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T10:38:24.421 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T10:38:24.421 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:24.421 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T10:38:24.421 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T10:38:24.422 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": false, 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2024-09-17T10:38:16.983429+0000", 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:24.423 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T10:38:24.424 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-09-17T10:38:24.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:24 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1433343376' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T10:38:25.019 INFO:teuthology.orchestra.run.smithi006.stderr:mgr not available, waiting (1/15)... 2024-09-17T10:38:27.593 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "ddf74116-74e0-11ef-bceb-c7b262605968", 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:27.594 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "smithi006" 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T10:38:27.595 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T10:38:27.599 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": false, 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T10:38:27.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2024-09-17T10:38:16.983429+0000", 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T10:38:27.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-09-17T10:38:27.906 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:27 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/80031743' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T10:38:28.217 INFO:teuthology.orchestra.run.smithi006.stderr:mgr not available, waiting (2/15)... 2024-09-17T10:38:30.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: Activating manager daemon smithi006.plhvdm 2024-09-17T10:38:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: mgrmap e2: smithi006.plhvdm(active, starting, since 0.00282805s) 2024-09-17T10:38:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T10:38:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T10:38:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T10:38:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:38:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:38:30.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: Manager daemon smithi006.plhvdm is now available 2024-09-17T10:38:30.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:38:30.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:38:30.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:30.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:30.204 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:29 smithi006 ceph-mon[23740]: from='mgr.14100 172.21.15.6:0/1741208837' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:30.720 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:30.721 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-09-17T10:38:30.721 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "ddf74116-74e0-11ef-bceb-c7b262605968", 2024-09-17T10:38:30.721 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2024-09-17T10:38:30.721 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T10:38:30.721 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T10:38:30.721 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T10:38:30.722 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:30.722 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T10:38:30.722 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T10:38:30.722 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2024-09-17T10:38:30.722 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:30.722 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T10:38:30.722 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "smithi006" 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:30.723 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T10:38:30.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": false, 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T10:38:30.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2024-09-17T10:38:16.983429+0000", 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T10:38:30.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-09-17T10:38:30.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:30 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/922000551' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T10:38:30.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:30 smithi006 ceph-mon[23740]: mgrmap e3: smithi006.plhvdm(active, since 1.00578s) 2024-09-17T10:38:31.373 INFO:teuthology.orchestra.run.smithi006.stderr:mgr not available, waiting (3/15)... 2024-09-17T10:38:33.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:33 smithi006 ceph-mon[23740]: mgrmap e4: smithi006.plhvdm(active, since 2s) 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "ddf74116-74e0-11ef-bceb-c7b262605968", 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T10:38:34.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "smithi006" 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T10:38:34.077 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T10:38:34.078 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T10:38:34.078 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T10:38:34.081 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T10:38:34.081 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T10:38:34.081 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:34.081 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T10:38:34.081 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T10:38:34.081 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T10:38:34.081 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T10:38:34.081 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T10:38:34.082 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2024-09-17T10:38:16.983429+0000", 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T10:38:34.083 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-09-17T10:38:34.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:34 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2409576210' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T10:38:34.630 INFO:teuthology.orchestra.run.smithi006.stderr:mgr is available 2024-09-17T10:38:35.213 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:35.216 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [global] 2024-09-17T10:38:35.216 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: fsid = ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:38:35.216 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T10:38:35.216 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T10:38:35.216 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T10:38:35.216 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T10:38:35.217 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:35.217 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [mgr] 2024-09-17T10:38:35.217 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T10:38:35.217 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-09-17T10:38:35.217 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [osd] 2024-09-17T10:38:35.217 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T10:38:35.217 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T10:38:35.728 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling cephadm module... 2024-09-17T10:38:35.850 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:35 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/395571916' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T10:38:36.869 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:36 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/190830111' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-17T10:38:37.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:37 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/190830111' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-17T10:38:37.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:37 smithi006 ceph-mon[23740]: mgrmap e5: smithi006.plhvdm(active, since 6s) 2024-09-17T10:38:37.888 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-09-17T10:38:37.889 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-17T10:38:37.889 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2024-09-17T10:38:37.889 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "active_name": "smithi006.plhvdm", 2024-09-17T10:38:37.889 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T10:38:37.889 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-09-17T10:38:38.407 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for the mgr to restart... 2024-09-17T10:38:38.407 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr epoch 5... 2024-09-17T10:38:38.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:38 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3084413451' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T10:38:43.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:42 smithi006 ceph-mon[23740]: Active manager daemon smithi006.plhvdm restarted 2024-09-17T10:38:43.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:42 smithi006 ceph-mon[23740]: Activating manager daemon smithi006.plhvdm 2024-09-17T10:38:43.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:42 smithi006 ceph-mon[23740]: osdmap e2: 0 total, 0 up, 0 in 2024-09-17T10:38:43.941 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-09-17T10:38:43.941 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-17T10:38:43.941 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T10:38:43.942 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-09-17T10:38:44.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: mgrmap e6: smithi006.plhvdm(active, starting, since 0.052644s) 2024-09-17T10:38:44.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:38:44.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:38:44.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: Manager daemon smithi006.plhvdm is now available 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:38:44.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:43 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:38:44.454 INFO:teuthology.orchestra.run.smithi006.stderr:mgr epoch 5 is available 2024-09-17T10:38:44.454 INFO:teuthology.orchestra.run.smithi006.stderr:Setting orchestrator backend to cephadm... 2024-09-17T10:38:45.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:44 smithi006 ceph-mon[23740]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T10:38:45.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:44 smithi006 ceph-mon[23740]: mgrmap e7: smithi006.plhvdm(active, since 1.06027s) 2024-09-17T10:38:45.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:44 smithi006 ceph-mon[23740]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T10:38:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: [17/Sep/2024:10:38:44] ENGINE Bus STARTING 2024-09-17T10:38:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: mgrmap e8: smithi006.plhvdm(active, since 2s) 2024-09-17T10:38:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: [17/Sep/2024:10:38:44] ENGINE Serving on https://172.21.15.6:7150 2024-09-17T10:38:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: [17/Sep/2024:10:38:44] ENGINE Bus STARTED 2024-09-17T10:38:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:38:46.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:46.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:46.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:45 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:38:46.282 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: value unchanged 2024-09-17T10:38:46.842 INFO:teuthology.orchestra.run.smithi006.stderr:Generating ssh key... 2024-09-17T10:38:47.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:47 smithi006 ceph-mon[23740]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:47.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:47 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:47.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:47 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:48.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:48 smithi006 ceph-mon[23740]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:48.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:48 smithi006 ceph-mon[23740]: Generating ssh key... 2024-09-17T10:38:48.714 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDCAjAdhXIdP8NkT5nZMmxzikj86Inybs1F8B1UXcc+d29slFRNTcYWE/6+dMoXGZkDtQA7+ciCYCehJ/UJmaw5R4+A4TlECBj0e1cg7QO9QK24DvHobjn5QeC7YqcKiiJzHJCBLswdO+thrKinG+2a33udV1k2z8YVMU/J9FbTDHl2/OQ7u/2B1k8RrrreF9TaeguCAVfQLAemac0jpBDQwubiqdIsy6PF62hxyyK00vQXVqKCj0z6HOP/7Z2D1/hVonlCYADS4MzU4qI//zAQbdYdGNywWLVwm2Y5EKusL2b/+JFZgr08vhHz1BjVIxlV6ADodQ1l9uVfeCqJfyaEYqeypmzi6PwFldBxw2nZBQsIJkshPol6TkypvlvhTgG+cZuEnRK+p56zgi4v74A8vdRqzTavdKEBV8yUfYamx9WQE+oeCK746rL8QtFr1jFXGOhalOWrOLZJNZk+UD66vFrQQ8UruUafxlXac6mytROkIMfV2GOFHmJ2CYWvyk= ceph-ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:38:49.231 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-17T10:38:49.231 INFO:teuthology.orchestra.run.smithi006.stderr:Adding key to root@localhost authorized_keys... 2024-09-17T10:38:49.240 INFO:teuthology.orchestra.run.smithi006.stderr:Adding host smithi006... 2024-09-17T10:38:49.892 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:49 smithi006 ceph-mon[23740]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:50.789 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Added host 'smithi006' with addr '172.21.15.6' 2024-09-17T10:38:50.801 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:50 smithi006 ceph-mon[23740]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi006", "addr": "172.21.15.6", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:50.801 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:50 smithi006 ceph-mon[23740]: Deploying cephadm binary to smithi006 2024-09-17T10:38:51.428 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying mon service with default placement... 2024-09-17T10:38:52.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:51 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:52.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:51 smithi006 ceph-mon[23740]: Added host smithi006 2024-09-17T10:38:52.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:51 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:38:52.158 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-17T10:38:53.032 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying mgr service with default placement... 2024-09-17T10:38:53.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:53 smithi006 ceph-mon[23740]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:53.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:53 smithi006 ceph-mon[23740]: Saving service mon spec with placement count:5 2024-09-17T10:38:53.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:53 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:54.236 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-17T10:38:54.817 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying crash service with default placement... 2024-09-17T10:38:55.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:55 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:55.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:55 smithi006 ceph-mon[23740]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:55.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:55 smithi006 ceph-mon[23740]: Saving service mgr spec with placement count:2 2024-09-17T10:38:55.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:55 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:55.446 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-17T10:38:56.023 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying prometheus service with default placement... 2024-09-17T10:38:56.622 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:56 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:56.622 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:56 smithi006 ceph-mon[23740]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:56.622 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:56 smithi006 ceph-mon[23740]: Saving service crash spec with placement * 2024-09-17T10:38:56.622 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:56 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:56.623 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-17T10:38:57.683 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying grafana service with default placement... 2024-09-17T10:38:57.871 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:57 smithi006 ceph-mon[23740]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:57.871 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:57 smithi006 ceph-mon[23740]: Saving service prometheus spec with placement count:1 2024-09-17T10:38:57.871 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:57 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:57.871 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:57 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:58.353 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-17T10:38:58.949 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying node-exporter service with default placement... 2024-09-17T10:38:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:59 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:59 smithi006 ceph-mon[23740]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:38:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:59 smithi006 ceph-mon[23740]: Saving service grafana spec with placement count:1 2024-09-17T10:38:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:38:59 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:38:59.568 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-17T10:39:00.492 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying alertmanager service with default placement... 2024-09-17T10:39:00.650 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:00 smithi006 ceph-mon[23740]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:39:00.650 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:00 smithi006 ceph-mon[23740]: Saving service node-exporter spec with placement * 2024-09-17T10:39:00.650 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:00 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:01.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-17T10:39:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:02 smithi006 ceph-mon[23740]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:39:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:02 smithi006 ceph-mon[23740]: Saving service alertmanager spec with placement count:1 2024-09-17T10:39:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:02 smithi006 ceph-mon[23740]: from='mgr.14120 172.21.15.6:0/799747141' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:04.223 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:03 smithi006 ceph-mon[23740]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:04.223 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:03 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1123564103' entity='client.admin' 2024-09-17T10:39:04.808 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling the dashboard module... 2024-09-17T10:39:05.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:05 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3908995711' entity='client.admin' 2024-09-17T10:39:06.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:06 smithi006 ceph-mon[23740]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:06.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:06 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2959449115' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-17T10:39:07.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:07 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2959449115' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-17T10:39:07.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:07 smithi006 ceph-mon[23740]: mgrmap e9: smithi006.plhvdm(active, since 23s) 2024-09-17T10:39:07.496 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-09-17T10:39:07.496 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-17T10:39:07.496 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2024-09-17T10:39:07.496 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "active_name": "smithi006.plhvdm", 2024-09-17T10:39:07.496 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T10:39:07.496 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-09-17T10:39:08.088 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for the mgr to restart... 2024-09-17T10:39:08.088 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr epoch 9... 2024-09-17T10:39:08.437 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:08 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/371704420' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T10:39:13.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: Active manager daemon smithi006.plhvdm restarted 2024-09-17T10:39:13.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: Activating manager daemon smithi006.plhvdm 2024-09-17T10:39:13.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: osdmap e3: 0 total, 0 up, 0 in 2024-09-17T10:39:13.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: mgrmap e10: smithi006.plhvdm(active, starting, since 0.0551565s) 2024-09-17T10:39:13.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:39:13.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:39:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T10:39:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T10:39:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T10:39:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: Manager daemon smithi006.plhvdm is now available 2024-09-17T10:39:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:39:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:39:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:39:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:13 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:39:13.675 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-09-17T10:39:13.675 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-17T10:39:13.676 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T10:39:13.676 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-09-17T10:39:14.380 INFO:teuthology.orchestra.run.smithi006.stderr:mgr epoch 9 is available 2024-09-17T10:39:14.381 INFO:teuthology.orchestra.run.smithi006.stderr:Generating a dashboard self-signed certificate... 2024-09-17T10:39:14.901 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:14 smithi006 ceph-mon[23740]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T10:39:14.901 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:14 smithi006 ceph-mon[23740]: mgrmap e11: smithi006.plhvdm(active, since 1.05762s) 2024-09-17T10:39:14.901 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:14 smithi006 ceph-mon[23740]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T10:39:14.901 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:14 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:15.107 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-17T10:39:15.730 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial admin user... 2024-09-17T10:39:16.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: [17/Sep/2024:10:39:14] ENGINE Bus STARTING 2024-09-17T10:39:16.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: [17/Sep/2024:10:39:14] ENGINE Serving on https://172.21.15.6:7150 2024-09-17T10:39:16.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: [17/Sep/2024:10:39:14] ENGINE Bus STARTED 2024-09-17T10:39:16.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:16.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:39:16.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:16.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:16.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:16.011 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:15 smithi006 ceph-mon[23740]: mgrmap e12: smithi006.plhvdm(active, since 2s) 2024-09-17T10:39:16.686 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$mkVL3l3gPP19xa5LbjQP6e4/wYEiywUdtie/n71kjlLNIg.i4OHcu", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726569556, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-17T10:39:17.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:17 smithi006 ceph-mon[23740]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:39:17.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:17 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:17.961 INFO:teuthology.orchestra.run.smithi006.stderr:Fetching dashboard port number... 2024-09-17T10:39:18.667 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 8443 2024-09-17T10:39:18.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:18 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/241622326' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-17T10:39:19.321 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: disabled 2024-09-17T10:39:19.328 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-09-17T10:39:19.329 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld.service is not enabled 2024-09-17T10:39:19.329 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-17T10:39:19.331 INFO:teuthology.orchestra.run.smithi006.stderr:Ceph Dashboard is now available at: 2024-09-17T10:39:19.332 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:39:19.332 INFO:teuthology.orchestra.run.smithi006.stderr: URL: https://smithi006.front.sepia.ceph.com:8443/ 2024-09-17T10:39:19.332 INFO:teuthology.orchestra.run.smithi006.stderr: User: admin 2024-09-17T10:39:19.332 INFO:teuthology.orchestra.run.smithi006.stderr: Password: l2udrhvucm 2024-09-17T10:39:19.332 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:39:19.332 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling autotune for osd_memory_target 2024-09-17T10:39:19.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:19 smithi006 ceph-mon[23740]: mgrmap e13: smithi006.plhvdm(active, since 6s) 2024-09-17T10:39:21.384 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-17T10:39:21.999 INFO:teuthology.orchestra.run.smithi006.stderr:You can access the Ceph CLI with: 2024-09-17T10:39:22.000 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:39:22.000 INFO:teuthology.orchestra.run.smithi006.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-17T10:39:22.000 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:39:22.000 INFO:teuthology.orchestra.run.smithi006.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-17T10:39:22.000 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:39:22.001 INFO:teuthology.orchestra.run.smithi006.stderr: ceph telemetry on 2024-09-17T10:39:22.001 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:39:22.001 INFO:teuthology.orchestra.run.smithi006.stderr:For more information see: 2024-09-17T10:39:22.001 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:39:22.001 INFO:teuthology.orchestra.run.smithi006.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-17T10:39:22.001 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T10:39:22.002 INFO:teuthology.orchestra.run.smithi006.stderr:Bootstrap complete. 2024-09-17T10:39:22.036 INFO:tasks.cephadm:Fetching config... 2024-09-17T10:39:22.037 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:39:22.037 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-17T10:39:22.055 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-17T10:39:22.055 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:39:22.055 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-17T10:39:22.111 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-17T10:39:22.112 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:39:22.112 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/keyring of=/dev/stdout 2024-09-17T10:39:22.178 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-17T10:39:22.178 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:39:22.178 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-17T10:39:22.234 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-17T10:39:22.234 DEBUG:teuthology.orchestra.run.smithi006:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDCAjAdhXIdP8NkT5nZMmxzikj86Inybs1F8B1UXcc+d29slFRNTcYWE/6+dMoXGZkDtQA7+ciCYCehJ/UJmaw5R4+A4TlECBj0e1cg7QO9QK24DvHobjn5QeC7YqcKiiJzHJCBLswdO+thrKinG+2a33udV1k2z8YVMU/J9FbTDHl2/OQ7u/2B1k8RrrreF9TaeguCAVfQLAemac0jpBDQwubiqdIsy6PF62hxyyK00vQXVqKCj0z6HOP/7Z2D1/hVonlCYADS4MzU4qI//zAQbdYdGNywWLVwm2Y5EKusL2b/+JFZgr08vhHz1BjVIxlV6ADodQ1l9uVfeCqJfyaEYqeypmzi6PwFldBxw2nZBQsIJkshPol6TkypvlvhTgG+cZuEnRK+p56zgi4v74A8vdRqzTavdKEBV8yUfYamx9WQE+oeCK746rL8QtFr1jFXGOhalOWrOLZJNZk+UD66vFrQQ8UruUafxlXac6mytROkIMfV2GOFHmJ2CYWvyk= ceph-ddf74116-74e0-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T10:39:22.321 INFO:teuthology.orchestra.run.smithi006.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDCAjAdhXIdP8NkT5nZMmxzikj86Inybs1F8B1UXcc+d29slFRNTcYWE/6+dMoXGZkDtQA7+ciCYCehJ/UJmaw5R4+A4TlECBj0e1cg7QO9QK24DvHobjn5QeC7YqcKiiJzHJCBLswdO+thrKinG+2a33udV1k2z8YVMU/J9FbTDHl2/OQ7u/2B1k8RrrreF9TaeguCAVfQLAemac0jpBDQwubiqdIsy6PF62hxyyK00vQXVqKCj0z6HOP/7Z2D1/hVonlCYADS4MzU4qI//zAQbdYdGNywWLVwm2Y5EKusL2b/+JFZgr08vhHz1BjVIxlV6ADodQ1l9uVfeCqJfyaEYqeypmzi6PwFldBxw2nZBQsIJkshPol6TkypvlvhTgG+cZuEnRK+p56zgi4v74A8vdRqzTavdKEBV8yUfYamx9WQE+oeCK746rL8QtFr1jFXGOhalOWrOLZJNZk+UD66vFrQQ8UruUafxlXac6mytROkIMfV2GOFHmJ2CYWvyk= ceph-ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:39:22.332 DEBUG:teuthology.orchestra.run.smithi149:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDCAjAdhXIdP8NkT5nZMmxzikj86Inybs1F8B1UXcc+d29slFRNTcYWE/6+dMoXGZkDtQA7+ciCYCehJ/UJmaw5R4+A4TlECBj0e1cg7QO9QK24DvHobjn5QeC7YqcKiiJzHJCBLswdO+thrKinG+2a33udV1k2z8YVMU/J9FbTDHl2/OQ7u/2B1k8RrrreF9TaeguCAVfQLAemac0jpBDQwubiqdIsy6PF62hxyyK00vQXVqKCj0z6HOP/7Z2D1/hVonlCYADS4MzU4qI//zAQbdYdGNywWLVwm2Y5EKusL2b/+JFZgr08vhHz1BjVIxlV6ADodQ1l9uVfeCqJfyaEYqeypmzi6PwFldBxw2nZBQsIJkshPol6TkypvlvhTgG+cZuEnRK+p56zgi4v74A8vdRqzTavdKEBV8yUfYamx9WQE+oeCK746rL8QtFr1jFXGOhalOWrOLZJNZk+UD66vFrQQ8UruUafxlXac6mytROkIMfV2GOFHmJ2CYWvyk= ceph-ddf74116-74e0-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T10:39:22.408 INFO:teuthology.orchestra.run.smithi149.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDCAjAdhXIdP8NkT5nZMmxzikj86Inybs1F8B1UXcc+d29slFRNTcYWE/6+dMoXGZkDtQA7+ciCYCehJ/UJmaw5R4+A4TlECBj0e1cg7QO9QK24DvHobjn5QeC7YqcKiiJzHJCBLswdO+thrKinG+2a33udV1k2z8YVMU/J9FbTDHl2/OQ7u/2B1k8RrrreF9TaeguCAVfQLAemac0jpBDQwubiqdIsy6PF62hxyyK00vQXVqKCj0z6HOP/7Z2D1/hVonlCYADS4MzU4qI//zAQbdYdGNywWLVwm2Y5EKusL2b/+JFZgr08vhHz1BjVIxlV6ADodQ1l9uVfeCqJfyaEYqeypmzi6PwFldBxw2nZBQsIJkshPol6TkypvlvhTgG+cZuEnRK+p56zgi4v74A8vdRqzTavdKEBV8yUfYamx9WQE+oeCK746rL8QtFr1jFXGOhalOWrOLZJNZk+UD66vFrQQ8UruUafxlXac6mytROkIMfV2GOFHmJ2CYWvyk= ceph-ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:39:22.421 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-17T10:39:22.591 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:22 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3741869610' entity='client.admin' 2024-09-17T10:39:25.145 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:24 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:25.145 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:24 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd/host:smithi006", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:39:25.145 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:24 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:25.145 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:24 smithi006 ceph-mon[23740]: Deploying daemon alertmanager.smithi006 on smithi006 2024-09-17T10:39:26.802 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-17T10:39:26.803 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-17T10:39:27.380 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:27 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1441427709' entity='client.admin' 2024-09-17T10:39:30.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:30 smithi006 ceph-mon[23740]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:39:30.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:30 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:30.809 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi149 2024-09-17T10:39:30.809 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-09-17T10:39:30.809 DEBUG:teuthology.orchestra.run.smithi149:> dd of=/etc/ceph/ceph.conf 2024-09-17T10:39:30.828 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-09-17T10:39:30.829 DEBUG:teuthology.orchestra.run.smithi149:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:39:30.887 INFO:tasks.cephadm:Adding host smithi149 to orchestrator... 2024-09-17T10:39:30.887 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch host add smithi149 2024-09-17T10:39:34.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:33 smithi006 ceph-mon[23740]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:35.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:35 smithi006 ceph-mon[23740]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:36.289 INFO:teuthology.orchestra.run.smithi006.stdout:Added host 'smithi149' with addr '172.21.15.149' 2024-09-17T10:39:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:36 smithi006 ceph-mon[23740]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi149", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:39:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:36 smithi006 ceph-mon[23740]: Deploying cephadm binary to smithi149 2024-09-17T10:39:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:36 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:36 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T10:39:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:36 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T10:39:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:36 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:39:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:36 smithi006 ceph-mon[23740]: Deploying daemon crash.smithi006 on smithi006 2024-09-17T10:39:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:36 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:37.899 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-17T10:39:38.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:37 smithi006 ceph-mon[23740]: Added host smithi149 2024-09-17T10:39:38.060 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:37 smithi006 ceph-mon[23740]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:38.060 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:37 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:39.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:39 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:39.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:39 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:39.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:39 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:39.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:39 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T10:39:39.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:39 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T10:39:39.478 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:39 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:39.478 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:39 smithi006 ceph-mon[23740]: Deploying daemon grafana.smithi006 on smithi006 2024-09-17T10:39:39.478 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:39 smithi006 ceph-mon[23740]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:40.145 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:39:40.145 INFO:teuthology.orchestra.run.smithi006.stdout:[{"addr": "172.21.15.6", "hostname": "smithi006", "labels": [], "status": ""}, {"addr": "172.21.15.149", "hostname": "smithi149", "labels": [], "status": ""}] 2024-09-17T10:39:40.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:40 smithi006 ceph-mon[23740]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:39:40.794 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-17T10:39:40.794 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-17T10:39:41.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:41 smithi006 ceph-mon[23740]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:43.726 INFO:teuthology.orchestra.run.smithi006.stderr:adjusted tunables profile to default 2024-09-17T10:39:44.048 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:43 smithi006 ceph-mon[23740]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:44.048 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:43 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:44.049 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:43 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/818543338' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-17T10:39:44.403 INFO:tasks.cephadm:Adding mon.smithi006 on smithi006 2024-09-17T10:39:44.403 INFO:tasks.cephadm:Adding mon.smithi149 on smithi149 2024-09-17T10:39:44.403 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi006:172.21.15.6=smithi006;smithi149:172.21.15.149=smithi149' 2024-09-17T10:39:45.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:44 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/818543338' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-17T10:39:45.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:44 smithi006 ceph-mon[23740]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T10:39:46.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:45 smithi006 ceph-mon[23740]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:46.121 INFO:teuthology.orchestra.run.smithi149.stdout:Scheduled mon update... 2024-09-17T10:39:46.769 DEBUG:teuthology.orchestra.run.smithi149:mon.smithi149> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi149.service 2024-09-17T10:39:46.771 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:39:46.771 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:39:47.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:47 smithi006 ceph-mon[23740]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi006:172.21.15.6=smithi006;smithi149:172.21.15.149=smithi149", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:39:47.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:47 smithi006 ceph-mon[23740]: Saving service mon spec with placement smithi006:172.21.15.6=smithi006;smithi149:172.21.15.149=smithi149;count:2 2024-09-17T10:39:47.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:47 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:48.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:48 smithi006 ceph-mon[23740]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:48.807 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:39:48.807 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:39:48.808 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:39:49.239 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:49 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/3967512108' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:39:50.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:50 smithi006 ceph-mon[23740]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:50.386 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:39:50.386 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:39:52.131 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:39:52.132 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:39:52.134 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:39:52.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:52 smithi006 ceph-mon[23740]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:52.345 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:52 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:52.345 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:52 smithi006 ceph-mon[23740]: Deploying daemon node-exporter.smithi006 on smithi006 2024-09-17T10:39:53.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:53 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/2911036629' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:39:53.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:53 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:53.782 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:39:53.783 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:39:54.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:54 smithi006 ceph-mon[23740]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:39:55.321 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:55 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' 2024-09-17T10:39:55.321 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:55 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-17T10:39:55.634 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:39:55.634 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:39:55.636 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:39:56.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:56 smithi006 ceph-mon[23740]: from='mgr.14162 172.21.15.6:0/3794581231' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-17T10:39:56.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:56 smithi006 ceph-mon[23740]: mgrmap e14: smithi006.plhvdm(active, since 42s) 2024-09-17T10:39:56.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:56 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/951632730' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:39:57.193 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:39:57.193 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:39:58.988 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:39:58.988 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:39:58.989 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:39:59.377 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:39:59 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/3096518170' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:40:00.634 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:40:00.634 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:40:01.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:01 smithi006 ceph-mon[23740]: Active manager daemon smithi006.plhvdm restarted 2024-09-17T10:40:01.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:01 smithi006 ceph-mon[23740]: Activating manager daemon smithi006.plhvdm 2024-09-17T10:40:01.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:01 smithi006 ceph-mon[23740]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T10:40:02.602 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:40:02.603 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:40:02.606 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:40:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: mgrmap e15: smithi006.plhvdm(active, starting, since 0.052447s) 2024-09-17T10:40:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:40:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: Manager daemon smithi006.plhvdm is now available 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:02.953 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:03.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:03 smithi006 ceph-mon[23740]: [17/Sep/2024:10:40:02] ENGINE Bus STARTING 2024-09-17T10:40:03.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:03 smithi006 ceph-mon[23740]: [17/Sep/2024:10:40:02] ENGINE Serving on https://172.21.15.6:7150 2024-09-17T10:40:03.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:03 smithi006 ceph-mon[23740]: [17/Sep/2024:10:40:02] ENGINE Bus STARTED 2024-09-17T10:40:03.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:03 smithi006 ceph-mon[23740]: mgrmap e16: smithi006.plhvdm(active, since 1.06015s) 2024-09-17T10:40:03.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:03 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/4123629092' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:40:03.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:03 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:04.286 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:40:04.286 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:40:05.447 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:05 smithi006 ceph-mon[23740]: mgrmap e17: smithi006.plhvdm(active, since 2s) 2024-09-17T10:40:05.447 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:05 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:40:06.211 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:40:06.214 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:40:06.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:06 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:06.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:06 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd/host:smithi006", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:40:06.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:06 smithi006 ceph-mon[23740]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:40:06.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:06 smithi006 ceph-mon[23740]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:40:06.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:06 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:06.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:06 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/4061940595' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:40:08.132 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:40:08.132 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:40:08.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:08.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:08.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:40:08.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:08.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T10:40:08.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T10:40:08.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:08.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:08 smithi006 ceph-mon[23740]: Deploying daemon crash.smithi149 on smithi149 2024-09-17T10:40:10.936 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:40:10.937 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:40:10.938 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:40:11.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:10 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/4175826517' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:40:12.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:12.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:40:12.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T10:40:12.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:40:12.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:12.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:12 smithi006 ceph-mon[23740]: Deploying daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:40:12.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:12.785 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:40:12.785 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:40:14.675 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:40:14.675 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:40:14.677 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:40:15.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:14 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/1674159900' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:40:16.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:16.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T10:40:16.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:16.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:15 smithi006 ceph-mon[23740]: Deploying daemon mon.smithi149 on smithi149 2024-09-17T10:40:16.382 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:40:16.382 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:40:17.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:19.720 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:40:19.720 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:38:13.266845Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T10:40:19.721 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-09-17T10:40:19.863 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:19 smithi149 ceph-mon[26044]: mon.smithi149@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-17T10:40:20.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:19 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/1189632165' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:40:20.204 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:19 smithi149 bash[25988]: a823656bce1e1bbe03b270f4044697515103f7711013a9267e709c100877a7f5 2024-09-17T10:40:20.204 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:19 smithi149 systemd[1]: Started Ceph mon.smithi149 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:40:21.512 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T10:40:21.513 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T10:40:25.116 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: mon.smithi006 calling monitor election 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/2048469535' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: mon.smithi149 calling monitor election 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:25.117 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: mon.smithi006 is new leader, mons smithi006,smithi149 in quorum (ranks 0,1) 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: monmap e2: 2 mons at {smithi006=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],smithi149=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0]} 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: fsmap 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: mgrmap e17: smithi006.plhvdm(active, since 23s) 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/2048469535' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: overall HEALTH_OK 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/2048469535' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:25.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:24 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/2048469535' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:40:25.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: mon.smithi006 calling monitor election 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/2048469535' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: mon.smithi149 calling monitor election 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: mon.smithi006 is new leader, mons smithi006,smithi149 in quorum (ranks 0,1) 2024-09-17T10:40:25.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: monmap e2: 2 mons at {smithi006=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],smithi149=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0]} 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: fsmap 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: mgrmap e17: smithi006.plhvdm(active, since 23s) 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/2048469535' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: overall HEALTH_OK 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/2048469535' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:25.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:24 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/2048469535' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:40:26.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:25 smithi006 ceph-mon[23740]: Deploying daemon node-exporter.smithi149 on smithi149 2024-09-17T10:40:26.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:25 smithi006 ceph-mon[23740]: mgrmap e18: smithi006.plhvdm(active, since 23s), standbys: smithi149.ijckag 2024-09-17T10:40:26.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:25 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi149.ijckag", "id": "smithi149.ijckag"}]: dispatch 2024-09-17T10:40:26.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:25 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:26.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:25 smithi149 ceph-mon[26044]: Deploying daemon node-exporter.smithi149 on smithi149 2024-09-17T10:40:26.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:25 smithi149 ceph-mon[26044]: mgrmap e18: smithi006.plhvdm(active, since 23s), standbys: smithi149.ijckag 2024-09-17T10:40:26.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:25 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi149.ijckag", "id": "smithi149.ijckag"}]: dispatch 2024-09-17T10:40:26.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:25 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:40:26.274 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:40:26.274 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":2,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","modified":"2024-09-17T10:40:19.803100Z","created":"2024-09-17T10:38:13.266845Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi149","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:3300","nonce":0},{"type":"v1","addr":"172.21.15.149:6789","nonce":0}]},"addr":"172.21.15.149:6789/0","public_addr":"172.21.15.149:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-17T10:40:26.275 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 2 2024-09-17T10:40:27.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:26 smithi006 ceph-mon[23740]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:27.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:26 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/4115098325' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:40:27.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:26 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:27.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:26 smithi149 ceph-mon[26044]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:27.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:26 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/4115098325' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T10:40:27.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:26 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:27.488 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-17T10:40:27.488 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-17T10:40:28.980 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:28 smithi006 ceph-mon[23740]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:28.980 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:28 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:28 smithi149 ceph-mon[26044]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:28 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:29.364 INFO:teuthology.orchestra.run.smithi006.stdout:# minimal ceph.conf for ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:40:29.364 INFO:teuthology.orchestra.run.smithi006.stdout:[global] 2024-09-17T10:40:29.365 INFO:teuthology.orchestra.run.smithi006.stdout: fsid = ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:40:29.365 INFO:teuthology.orchestra.run.smithi006.stdout: mon_host = [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] 2024-09-17T10:40:29.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:29 smithi006 ceph-mon[23740]: Deploying daemon prometheus.smithi006 on smithi006 2024-09-17T10:40:29.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:29 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1076358107' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:30.192 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-17T10:40:30.193 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:40:30.193 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T10:40:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:29 smithi149 ceph-mon[26044]: Deploying daemon prometheus.smithi006 on smithi006 2024-09-17T10:40:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:29 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1076358107' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:30.227 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:40:30.228 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:40:30.306 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-09-17T10:40:30.306 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T10:40:30.339 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-09-17T10:40:30.340 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:40:30.410 DEBUG:tasks.cephadm:set 0 configs 2024-09-17T10:40:30.410 INFO:tasks.cephadm:Deploying OSDs... 2024-09-17T10:40:30.410 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:40:30.410 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T10:40:30.434 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T10:40:30.434 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_1 2024-09-17T10:40:30.496 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T10:40:30.496 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T10:40:30.496 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-17T10:40:30.496 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T10:40:30.496 INFO:teuthology.orchestra.run.smithi006.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T10:40:30.496 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2024-09-17 10:39:51.676271680 +0000 2024-09-17T10:40:30.496 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2024-09-17 10:37:14.777392054 +0000 2024-09-17T10:40:30.497 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2024-09-17 10:37:14.777392054 +0000 2024-09-17T10:40:30.497 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: 2024-09-17 10:37:14.777392054 +0000 2024-09-17T10:40:30.497 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T10:40:30.573 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2024-09-17T10:40:30.573 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2024-09-17T10:40:30.573 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.00021121 s, 2.4 MB/s 2024-09-17T10:40:30.575 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T10:40:30.639 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_2 2024-09-17T10:40:30.696 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T10:40:30.696 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T10:40:30.696 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-17T10:40:30.696 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T10:40:30.696 INFO:teuthology.orchestra.run.smithi006.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T10:40:30.697 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2024-09-17 10:39:51.677271669 +0000 2024-09-17T10:40:30.697 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2024-09-17 10:37:15.116388431 +0000 2024-09-17T10:40:30.697 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2024-09-17 10:37:15.116388431 +0000 2024-09-17T10:40:30.697 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: 2024-09-17 10:37:15.116388431 +0000 2024-09-17T10:40:30.697 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T10:40:30.766 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2024-09-17T10:40:30.766 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2024-09-17T10:40:30.767 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000215559 s, 2.4 MB/s 2024-09-17T10:40:30.768 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T10:40:30.825 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_3 2024-09-17T10:40:30.881 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T10:40:30.882 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T10:40:30.882 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-17T10:40:30.882 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T10:40:30.882 INFO:teuthology.orchestra.run.smithi006.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T10:40:30.882 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2024-09-17 10:39:51.677271669 +0000 2024-09-17T10:40:30.882 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2024-09-17 10:37:15.453384831 +0000 2024-09-17T10:40:30.882 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2024-09-17 10:37:15.453384831 +0000 2024-09-17T10:40:30.882 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: 2024-09-17 10:37:15.453384831 +0000 2024-09-17T10:40:30.882 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T10:40:30.947 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2024-09-17T10:40:30.947 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2024-09-17T10:40:30.947 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000232278 s, 2.2 MB/s 2024-09-17T10:40:30.948 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T10:40:31.005 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_4 2024-09-17T10:40:31.062 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T10:40:31.062 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T10:40:31.062 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-17T10:40:31.062 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T10:40:31.063 INFO:teuthology.orchestra.run.smithi006.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T10:40:31.063 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2024-09-17 10:39:51.677271669 +0000 2024-09-17T10:40:31.063 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2024-09-17 10:37:15.781381326 +0000 2024-09-17T10:40:31.063 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2024-09-17 10:37:15.781381326 +0000 2024-09-17T10:40:31.063 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: 2024-09-17 10:37:15.781381326 +0000 2024-09-17T10:40:31.063 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T10:40:31.125 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:30 smithi006 ceph-mon[23740]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:31.127 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2024-09-17T10:40:31.128 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2024-09-17T10:40:31.128 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000233788 s, 2.2 MB/s 2024-09-17T10:40:31.129 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T10:40:31.186 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-09-17T10:40:31.186 DEBUG:teuthology.orchestra.run.smithi149:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T10:40:31.205 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T10:40:31.206 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_1 2024-09-17T10:40:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:30 smithi149 ceph-mon[26044]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:31.223 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T10:40:31.223 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T10:40:31.223 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 5h/5d Inode: 797 Links: 1 2024-09-17T10:40:31.224 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T10:40:31.224 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T10:40:31.224 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2024-09-17 10:40:25.440524874 +0000 2024-09-17T10:40:31.224 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2024-09-17 10:37:22.790002931 +0000 2024-09-17T10:40:31.224 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2024-09-17 10:37:22.790002931 +0000 2024-09-17T10:40:31.224 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: 2024-09-17 10:37:22.790002931 +0000 2024-09-17T10:40:31.224 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T10:40:31.290 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2024-09-17T10:40:31.291 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2024-09-17T10:40:31.291 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000239266 s, 2.1 MB/s 2024-09-17T10:40:31.292 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T10:40:31.350 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_2 2024-09-17T10:40:31.408 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T10:40:31.409 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T10:40:31.409 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 5h/5d Inode: 818 Links: 1 2024-09-17T10:40:31.409 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T10:40:31.409 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T10:40:31.409 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2024-09-17 10:40:25.440524874 +0000 2024-09-17T10:40:31.410 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2024-09-17 10:37:23.105996666 +0000 2024-09-17T10:40:31.410 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2024-09-17 10:37:23.105996666 +0000 2024-09-17T10:40:31.410 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: 2024-09-17 10:37:23.105996666 +0000 2024-09-17T10:40:31.411 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T10:40:31.474 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2024-09-17T10:40:31.475 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2024-09-17T10:40:31.475 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000222285 s, 2.3 MB/s 2024-09-17T10:40:31.476 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T10:40:31.534 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_3 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 5h/5d Inode: 837 Links: 1 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2024-09-17 10:40:25.441524854 +0000 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2024-09-17 10:37:23.431990202 +0000 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2024-09-17 10:37:23.431990202 +0000 2024-09-17T10:40:31.591 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: 2024-09-17 10:37:23.431990202 +0000 2024-09-17T10:40:31.592 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T10:40:31.656 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2024-09-17T10:40:31.656 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2024-09-17T10:40:31.656 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.00017748 s, 2.9 MB/s 2024-09-17T10:40:31.657 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T10:40:31.714 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_4 2024-09-17T10:40:31.770 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T10:40:31.770 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T10:40:31.771 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 5h/5d Inode: 855 Links: 1 2024-09-17T10:40:31.771 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T10:40:31.771 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T10:40:31.771 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2024-09-17 10:40:25.441524854 +0000 2024-09-17T10:40:31.771 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2024-09-17 10:37:23.771983460 +0000 2024-09-17T10:40:31.771 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2024-09-17 10:37:23.771983460 +0000 2024-09-17T10:40:31.771 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: 2024-09-17 10:37:23.771983460 +0000 2024-09-17T10:40:31.771 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T10:40:31.835 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2024-09-17T10:40:31.835 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2024-09-17T10:40:31.835 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000226093 s, 2.3 MB/s 2024-09-17T10:40:31.837 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T10:40:31.894 INFO:tasks.cephadm:Deploying osd.0 on smithi006 with /dev/vg_nvme/lv_4... 2024-09-17T10:40:31.895 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T10:40:32.822 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:32 smithi006 ceph-mon[23740]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:32.822 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:32 smithi149 ceph-mon[26044]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:34.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:34 smithi006 ceph-mon[23740]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:34 smithi149 ceph-mon[26044]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:36.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:36 smithi006 ceph-mon[23740]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:36.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:36 smithi149 ceph-mon[26044]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:38.615 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:38 smithi006 ceph-mon[23740]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:38 smithi149 ceph-mon[26044]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:40.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:40 smithi006 ceph-mon[23740]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:40.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:40 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:40.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:40 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:40:40.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:40 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:40.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:40 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:40:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:40 smithi149 ceph-mon[26044]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:40 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:40 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:40:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:40 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:40.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:40 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:40:42.330 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:40:42.349 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi006:vg_nvme/lv_4 2024-09-17T10:40:42.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:42 smithi006 ceph-mon[23740]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:42.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:42 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:42.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:42 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:42.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:42 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:42.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:42 smithi149 ceph-mon[26044]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:42.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:42 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:42.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:42 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:42.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:42 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:43.909 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:43 smithi006 ceph-mon[23740]: Updating smithi149:/etc/ceph/ceph.conf 2024-09-17T10:40:43.909 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:43 smithi006 ceph-mon[23740]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:40:43.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:43 smithi149 ceph-mon[26044]: Updating smithi149:/etc/ceph/ceph.conf 2024-09-17T10:40:43.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:43 smithi149 ceph-mon[26044]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:40:44.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:44 smithi006 ceph-mon[23740]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:44 smithi149 ceph-mon[26044]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:46.696 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:46 smithi006 ceph-mon[23740]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:46.697 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:46 smithi006 ceph-mon[23740]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:40:46.697 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:40:46.697 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:40:46.697 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:46.697 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:46.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:46 smithi149 ceph-mon[26044]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:46.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:46 smithi149 ceph-mon[26044]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:40:46.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:40:46.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:40:46.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:46.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:47.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:47 smithi006 ceph-mon[23740]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:40:47.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:47.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:47.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:47 smithi006 ceph-mon[23740]: Reconfiguring mon.smithi006 (unknown last config time)... 2024-09-17T10:40:47.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T10:40:47.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T10:40:47.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:47.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:47 smithi006 ceph-mon[23740]: Reconfiguring daemon mon.smithi006 on smithi006 2024-09-17T10:40:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:47 smithi149 ceph-mon[26044]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:40:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:47 smithi149 ceph-mon[26044]: Reconfiguring mon.smithi006 (unknown last config time)... 2024-09-17T10:40:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T10:40:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T10:40:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:47 smithi149 ceph-mon[26044]: Reconfiguring daemon mon.smithi006 on smithi006 2024-09-17T10:40:48.949 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:48 smithi006 ceph-mon[23740]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:48.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:48 smithi149 ceph-mon[26044]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:50.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:50.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:49 smithi006 ceph-mon[23740]: Reconfiguring mgr.smithi006.plhvdm (unknown last config time)... 2024-09-17T10:40:50.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.plhvdm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:40:50.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:40:50.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:50.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:49 smithi006 ceph-mon[23740]: Reconfiguring daemon mgr.smithi006.plhvdm on smithi006 2024-09-17T10:40:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:49 smithi149 ceph-mon[26044]: Reconfiguring mgr.smithi006.plhvdm (unknown last config time)... 2024-09-17T10:40:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.plhvdm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:40:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:40:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:40:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:49 smithi149 ceph-mon[26044]: Reconfiguring daemon mgr.smithi006.plhvdm on smithi006 2024-09-17T10:40:51.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:50 smithi006 ceph-mon[23740]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:51.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:50 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1454621383' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2fbbec5b-85fb-4bd7-82ef-adc149c87960"}]: dispatch 2024-09-17T10:40:51.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:50 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1454621383' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2fbbec5b-85fb-4bd7-82ef-adc149c87960"}]': finished 2024-09-17T10:40:51.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:50 smithi006 ceph-mon[23740]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T10:40:51.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:50 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:40:51.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:50 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3888904233' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:40:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:50 smithi149 ceph-mon[26044]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:50 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1454621383' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2fbbec5b-85fb-4bd7-82ef-adc149c87960"}]: dispatch 2024-09-17T10:40:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:50 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1454621383' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2fbbec5b-85fb-4bd7-82ef-adc149c87960"}]': finished 2024-09-17T10:40:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:50 smithi149 ceph-mon[26044]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T10:40:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:50 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:40:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:50 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3888904233' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:40:52.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:52 smithi006 ceph-mon[23740]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:52 smithi149 ceph-mon[26044]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:54.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:54 smithi006 ceph-mon[23740]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:54.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:54 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:54 smithi149 ceph-mon[26044]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:54 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:40:56.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:55 smithi006 ceph-mon[23740]: Reconfiguring alertmanager.smithi006 (dependencies changed)... 2024-09-17T10:40:56.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:55 smithi006 ceph-mon[23740]: Reconfiguring daemon alertmanager.smithi006 on smithi006 2024-09-17T10:40:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:55 smithi149 ceph-mon[26044]: Reconfiguring alertmanager.smithi006 (dependencies changed)... 2024-09-17T10:40:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:55 smithi149 ceph-mon[26044]: Reconfiguring daemon alertmanager.smithi006 on smithi006 2024-09-17T10:40:57.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:56 smithi006 ceph-mon[23740]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:57.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:56 smithi149 ceph-mon[26044]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:59.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:40:58 smithi006 ceph-mon[23740]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:40:59.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:40:58 smithi149 ceph-mon[26044]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:01.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:01 smithi006 ceph-mon[23740]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:01.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:01.293 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:01 smithi006 ceph-mon[23740]: Reconfiguring crash.smithi006 (monmap changed)... 2024-09-17T10:41:01.293 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T10:41:01.293 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:01.293 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:01 smithi006 ceph-mon[23740]: Reconfiguring daemon crash.smithi006 on smithi006 2024-09-17T10:41:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:01 smithi149 ceph-mon[26044]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:01 smithi149 ceph-mon[26044]: Reconfiguring crash.smithi006 (monmap changed)... 2024-09-17T10:41:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T10:41:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:01 smithi149 ceph-mon[26044]: Reconfiguring daemon crash.smithi006 on smithi006 2024-09-17T10:41:02.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T10:41:02.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:02.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:41:02.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:41:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T10:41:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:41:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:41:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:03 smithi149 ceph-mon[26044]: Deploying daemon osd.0 on smithi006 2024-09-17T10:41:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:03 smithi149 ceph-mon[26044]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:03 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:03.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:03 smithi006 ceph-mon[23740]: Deploying daemon osd.0 on smithi006 2024-09-17T10:41:03.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:03 smithi006 ceph-mon[23740]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:03.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:03 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:04.248 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:04 smithi006 ceph-mon[23740]: Reconfiguring grafana.smithi006 (dependencies changed)... 2024-09-17T10:41:04.248 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:04 smithi006 ceph-mon[23740]: Reconfiguring daemon grafana.smithi006 on smithi006 2024-09-17T10:41:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:04 smithi149 ceph-mon[26044]: Reconfiguring grafana.smithi006 (dependencies changed)... 2024-09-17T10:41:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:04 smithi149 ceph-mon[26044]: Reconfiguring daemon grafana.smithi006 on smithi006 2024-09-17T10:41:05.298 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:05 smithi006 ceph-mon[23740]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:05 smithi149 ceph-mon[26044]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:07.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:07 smithi006 ceph-mon[23740]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:07 smithi149 ceph-mon[26044]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:08.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:08 smithi006 ceph-mon[23740]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:08.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:08 smithi149 ceph-mon[26044]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:11.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:10 smithi006 ceph-mon[23740]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:10 smithi149 ceph-mon[26044]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:12.116 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:11 smithi006 ceph-mon[23740]: from='osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T10:41:12.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:11 smithi149 ceph-mon[26044]: from='osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T10:41:12.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:12 smithi006 ceph-mon[23740]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:12.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:12 smithi006 ceph-mon[23740]: from='osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T10:41:12.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:12 smithi006 ceph-mon[23740]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T10:41:12.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:12.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:12 smithi006 ceph-mon[23740]: from='osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:41:12.994 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:12.994 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T10:41:12.994 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:13.088 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:12 smithi149 ceph-mon[26044]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:13.088 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:12 smithi149 ceph-mon[26044]: from='osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T10:41:13.088 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:12 smithi149 ceph-mon[26044]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T10:41:13.088 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:12 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:13.088 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:12 smithi149 ceph-mon[26044]: from='osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:41:13.088 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:12 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:13.088 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:12 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T10:41:13.088 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:12 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:14.187 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:13 smithi149 ceph-mon[26044]: Reconfiguring crash.smithi149 (monmap changed)... 2024-09-17T10:41:14.188 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:13 smithi149 ceph-mon[26044]: Reconfiguring daemon crash.smithi149 on smithi149 2024-09-17T10:41:14.188 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:13 smithi149 ceph-mon[26044]: from='osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-09-17T10:41:14.188 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:13 smithi149 ceph-mon[26044]: osdmap e8: 1 total, 0 up, 1 in 2024-09-17T10:41:14.188 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:13 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:14.188 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:13 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:14.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:13 smithi006 ceph-mon[23740]: Reconfiguring crash.smithi149 (monmap changed)... 2024-09-17T10:41:14.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:13 smithi006 ceph-mon[23740]: Reconfiguring daemon crash.smithi149 on smithi149 2024-09-17T10:41:14.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:13 smithi006 ceph-mon[23740]: from='osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-09-17T10:41:14.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:13 smithi006 ceph-mon[23740]: osdmap e8: 1 total, 0 up, 1 in 2024-09-17T10:41:14.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:13 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:14.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:13 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:14.278 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 0 on host 'smithi006' 2024-09-17T10:41:14.856 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: purged_snaps scrub starts 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: purged_snaps scrub ok 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843] boot 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: Reconfiguring mgr.smithi149.ijckag (monmap changed)... 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: Reconfiguring daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:41:14.857 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:14.887 DEBUG:teuthology.orchestra.run.smithi006:osd.0> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.0.service 2024-09-17T10:41:14.889 INFO:tasks.cephadm:Deploying osd.1 on smithi006 with /dev/vg_nvme/lv_3... 2024-09-17T10:41:14.889 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T10:41:15.094 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: purged_snaps scrub starts 2024-09-17T10:41:15.094 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: purged_snaps scrub ok 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: osd.0 [v2:172.21.15.6:6802/3722513843,v1:172.21.15.6:6803/3722513843] boot 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: Reconfiguring mgr.smithi149.ijckag (monmap changed)... 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: Reconfiguring daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:41:15.095 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:15.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:15 smithi149 ceph-mon[26044]: osdmap e10: 1 total, 1 up, 1 in 2024-09-17T10:41:15.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:15.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:15 smithi149 ceph-mon[26044]: Reconfiguring mon.smithi149 (monmap changed)... 2024-09-17T10:41:15.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T10:41:15.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T10:41:15.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:15.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:15 smithi149 ceph-mon[26044]: Reconfiguring daemon mon.smithi149 on smithi149 2024-09-17T10:41:16.146 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:15 smithi006 ceph-mon[23740]: osdmap e10: 1 total, 1 up, 1 in 2024-09-17T10:41:16.146 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:16.146 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:15 smithi006 ceph-mon[23740]: Reconfiguring mon.smithi149 (monmap changed)... 2024-09-17T10:41:16.146 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T10:41:16.147 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T10:41:16.147 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:16.147 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:15 smithi006 ceph-mon[23740]: Reconfiguring daemon mon.smithi149 on smithi149 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.6:9093"}]: dispatch 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.6:3000"}]: dispatch 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.6:9095"}]: dispatch 2024-09-17T10:41:17.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:17.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:41:17.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:17.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:41:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T10:41:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.6:9093"}]: dispatch 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.6:3000"}]: dispatch 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.6:9095"}]: dispatch 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:17.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:41:17.729 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:41:17.753 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi006:vg_nvme/lv_3 2024-09-17T10:41:18.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:18 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T10:41:18.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:18 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.6:9093"}]: dispatch 2024-09-17T10:41:18.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:18 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:41:18.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:18 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.6:3000"}]: dispatch 2024-09-17T10:41:18.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:18 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:41:18.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:18 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.6:9095"}]: dispatch 2024-09-17T10:41:18.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:18 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:18.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:18 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T10:41:18.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:18 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.6:9093"}]: dispatch 2024-09-17T10:41:18.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:18 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:41:18.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:18 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.6:3000"}]: dispatch 2024-09-17T10:41:18.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:18 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:41:18.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:18 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.6:9095"}]: dispatch 2024-09-17T10:41:18.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:18 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:19.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:19 smithi006 ceph-mon[23740]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:19 smithi149 ceph-mon[26044]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:20.078 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:20 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:41:20.079 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:20 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:41:20.079 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:20 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:20.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:20 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:41:20.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:20 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:41:20.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:20 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:21.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:21 smithi006 ceph-mon[23740]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:21.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:21 smithi006 ceph-mon[23740]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:41:21.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:21 smithi149 ceph-mon[26044]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:21.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:21 smithi149 ceph-mon[26044]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:41:23.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:23 smithi149 ceph-mon[26044]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:23.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:23 smithi006 ceph-mon[23740]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:24.353 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:24 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/41565809' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "927193ac-f68c-463b-b254-a0096ab97bbb"}]: dispatch 2024-09-17T10:41:24.353 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:24 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/41565809' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "927193ac-f68c-463b-b254-a0096ab97bbb"}]': finished 2024-09-17T10:41:24.353 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:24 smithi006 ceph-mon[23740]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T10:41:24.353 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:24.353 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:24.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:24 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/41565809' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "927193ac-f68c-463b-b254-a0096ab97bbb"}]: dispatch 2024-09-17T10:41:24.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:24 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/41565809' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "927193ac-f68c-463b-b254-a0096ab97bbb"}]': finished 2024-09-17T10:41:24.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:24 smithi149 ceph-mon[26044]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T10:41:24.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:24.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:25.383 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:25 smithi149 ceph-mon[26044]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:25.384 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:25 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/881262649' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:41:25.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:25 smithi006 ceph-mon[23740]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:25.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:25 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/881262649' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:41:27.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:27 smithi006 ceph-mon[23740]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:27 smithi149 ceph-mon[26044]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:29.247 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:29 smithi006 ceph-mon[23740]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:29.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:29 smithi149 ceph-mon[26044]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:30.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: Detected new or changed devices on smithi006 2024-09-17T10:41:30.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:30.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:41:30.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi006 to 12868M 2024-09-17T10:41:30.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:30.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:30.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:41:30.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:30.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:41:30.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:30 smithi006 ceph-mon[23740]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: Detected new or changed devices on smithi006 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi006 to 12868M 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:41:30.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:30 smithi149 ceph-mon[26044]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:32.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:32.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T10:41:32.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T10:41:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:33 smithi149 ceph-mon[26044]: Deploying daemon osd.1 on smithi006 2024-09-17T10:41:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:33 smithi149 ceph-mon[26044]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:33.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:33 smithi006 ceph-mon[23740]: Deploying daemon osd.1 on smithi006 2024-09-17T10:41:33.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:33 smithi006 ceph-mon[23740]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:35 smithi149 ceph-mon[26044]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:35.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:35 smithi006 ceph-mon[23740]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:37.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:37 smithi006 ceph-mon[23740]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:37.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:37 smithi149 ceph-mon[26044]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:38.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:38 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:38.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:38 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:38.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:38 smithi006 ceph-mon[23740]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:38.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:38 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:38.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:38 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:38.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:38 smithi149 ceph-mon[26044]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:39.639 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:39 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:39.639 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:39 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:41:39.639 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:39 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:39.640 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:39 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:41:39.641 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:39 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:39.641 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:39 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:41:39.641 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:39 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:39.641 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:39 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:41:41.432 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:41 smithi006 ceph-mon[23740]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:41.432 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:41.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:41 smithi149 ceph-mon[26044]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:41.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:41.753 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 1 on host 'smithi006' 2024-09-17T10:41:42.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:42 smithi006 ceph-mon[23740]: from='osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T10:41:42.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:42 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:42.369 DEBUG:teuthology.orchestra.run.smithi006:osd.1> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.1.service 2024-09-17T10:41:42.372 INFO:tasks.cephadm:Deploying osd.2 on smithi006 with /dev/vg_nvme/lv_2... 2024-09-17T10:41:42.372 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T10:41:42.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:42 smithi149 ceph-mon[26044]: from='osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T10:41:42.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:42 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:43 smithi149 ceph-mon[26044]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:43 smithi149 ceph-mon[26044]: from='osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T10:41:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:43 smithi149 ceph-mon[26044]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T10:41:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:43 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:43 smithi149 ceph-mon[26044]: from='osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:41:43.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:43 smithi006 ceph-mon[23740]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:43.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:43 smithi006 ceph-mon[23740]: from='osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T10:41:43.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:43 smithi006 ceph-mon[23740]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T10:41:43.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:43 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:43.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:43 smithi006 ceph-mon[23740]: from='osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:41:43.452 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:41:43 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[38294]: 2024-09-17T10:41:43.088+0000 7f006979a700 -1 osd.1 0 waiting for initial osdmap 2024-09-17T10:41:43.452 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:41:43 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[38294]: 2024-09-17T10:41:43.092+0000 7f006092a700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:41:44.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:44 smithi006 ceph-mon[23740]: from='osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-09-17T10:41:44.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:44 smithi006 ceph-mon[23740]: osdmap e13: 2 total, 1 up, 2 in 2024-09-17T10:41:44.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:44 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:44.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:44 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:44 smithi149 ceph-mon[26044]: from='osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-09-17T10:41:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:44 smithi149 ceph-mon[26044]: osdmap e13: 2 total, 1 up, 2 in 2024-09-17T10:41:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:44 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:44 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:45.295 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:45 smithi006 ceph-mon[23740]: purged_snaps scrub starts 2024-09-17T10:41:45.295 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:45 smithi006 ceph-mon[23740]: purged_snaps scrub ok 2024-09-17T10:41:45.295 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:45 smithi006 ceph-mon[23740]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:45.295 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:45 smithi006 ceph-mon[23740]: osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025] boot 2024-09-17T10:41:45.295 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:45 smithi006 ceph-mon[23740]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T10:41:45.296 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:45 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:45.301 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:41:45.325 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi006:vg_nvme/lv_2 2024-09-17T10:41:45.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:45 smithi149 ceph-mon[26044]: purged_snaps scrub starts 2024-09-17T10:41:45.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:45 smithi149 ceph-mon[26044]: purged_snaps scrub ok 2024-09-17T10:41:45.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:45 smithi149 ceph-mon[26044]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T10:41:45.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:45 smithi149 ceph-mon[26044]: osd.1 [v2:172.21.15.6:6810/3766560025,v1:172.21.15.6:6811/3766560025] boot 2024-09-17T10:41:45.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:45 smithi149 ceph-mon[26044]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T10:41:45.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:45 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:41:46.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:46 smithi006 ceph-mon[23740]: osdmap e15: 2 total, 2 up, 2 in 2024-09-17T10:41:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:46 smithi149 ceph-mon[26044]: osdmap e15: 2 total, 2 up, 2 in 2024-09-17T10:41:47.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:47 smithi006 ceph-mon[23740]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:47 smithi149 ceph-mon[26044]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:49.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:49 smithi006 ceph-mon[23740]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:49.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:49.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:41:49.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:41:49.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:49 smithi149 ceph-mon[26044]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:41:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:41:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:50.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:50 smithi006 ceph-mon[23740]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:41:50.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:50 smithi149 ceph-mon[26044]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:41:51.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:51 smithi006 ceph-mon[23740]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:51.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:51 smithi149 ceph-mon[26044]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:52.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:52.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:52.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:41:52.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:41:52.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi006 to 6434M 2024-09-17T10:41:52.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:52.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:52.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:41:52.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:52.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:41:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:52.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:41:52.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:41:52.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi006 to 6434M 2024-09-17T10:41:52.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:52.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:52.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:41:52.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:41:52.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:41:53.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:53 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1268310874' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef897e3f-bf8c-4430-a3a6-ead5208c6a51"}]: dispatch 2024-09-17T10:41:53.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:53 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1268310874' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef897e3f-bf8c-4430-a3a6-ead5208c6a51"}]': finished 2024-09-17T10:41:53.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:53 smithi006 ceph-mon[23740]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T10:41:53.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:53 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:41:53.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:53 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1268310874' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef897e3f-bf8c-4430-a3a6-ead5208c6a51"}]: dispatch 2024-09-17T10:41:53.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:53 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1268310874' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef897e3f-bf8c-4430-a3a6-ead5208c6a51"}]': finished 2024-09-17T10:41:53.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:53 smithi149 ceph-mon[26044]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T10:41:53.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:53 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:41:54.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:54 smithi006 ceph-mon[23740]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:54.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:54 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/4008449338' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:41:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:54 smithi149 ceph-mon[26044]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:54 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/4008449338' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:41:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:56 smithi006 ceph-mon[23740]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:56 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:56 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:56.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:56 smithi149 ceph-mon[26044]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:56.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:56 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:56.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:56 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:41:58.788 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:41:58 smithi006 ceph-mon[23740]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:41:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:41:58 smithi149 ceph-mon[26044]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:00.808 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:00 smithi006 ceph-mon[23740]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:00.809 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:00 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T10:42:00.809 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:00 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:00.809 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:00 smithi006 ceph-mon[23740]: Deploying daemon osd.2 on smithi006 2024-09-17T10:42:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:00 smithi149 ceph-mon[26044]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:00 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T10:42:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:00 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:00 smithi149 ceph-mon[26044]: Deploying daemon osd.2 on smithi006 2024-09-17T10:42:02.039 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:42:02.039 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:42:02.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:42:02.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:42:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:02 smithi006 ceph-mon[23740]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:02 smithi149 ceph-mon[26044]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:04.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:04 smithi006 ceph-mon[23740]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:05.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:04 smithi149 ceph-mon[26044]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:06.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:05 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:06.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:05 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:06.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:05 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:06.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:05 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:06.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:05 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:06.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:05 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:06.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:05 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:06.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:05 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:07.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:06 smithi006 ceph-mon[23740]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:07.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:06 smithi149 ceph-mon[26044]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:08.092 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 2 on host 'smithi006' 2024-09-17T10:42:08.960 DEBUG:teuthology.orchestra.run.smithi006:osd.2> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.2.service 2024-09-17T10:42:08.962 INFO:tasks.cephadm:Deploying osd.3 on smithi006 with /dev/vg_nvme/lv_1... 2024-09-17T10:42:08.963 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T10:42:09.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:08 smithi006 ceph-mon[23740]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:09.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:08 smithi006 ceph-mon[23740]: from='osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T10:42:09.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:08 smithi149 ceph-mon[26044]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:08 smithi149 ceph-mon[26044]: from='osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T10:42:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:10.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:09 smithi006 ceph-mon[23740]: from='osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T10:42:10.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:09 smithi006 ceph-mon[23740]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T10:42:10.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:09 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:10.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:09 smithi006 ceph-mon[23740]: from='osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:42:10.202 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:42:09 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[41963]: 2024-09-17T10:42:09.939+0000 7f7371739700 -1 osd.2 0 waiting for initial osdmap 2024-09-17T10:42:10.202 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:42:09 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[41963]: 2024-09-17T10:42:09.946+0000 7f736e0d4700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:42:10.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:09 smithi149 ceph-mon[26044]: from='osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T10:42:10.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:09 smithi149 ceph-mon[26044]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T10:42:10.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:09 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:10.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:09 smithi149 ceph-mon[26044]: from='osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:42:11.188 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:10 smithi006 ceph-mon[23740]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:11.188 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:10 smithi006 ceph-mon[23740]: from='osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-09-17T10:42:11.188 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:10 smithi006 ceph-mon[23740]: osdmap e18: 3 total, 2 up, 3 in 2024-09-17T10:42:11.188 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:11.189 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:10 smithi149 ceph-mon[26044]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:10 smithi149 ceph-mon[26044]: from='osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-09-17T10:42:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:10 smithi149 ceph-mon[26044]: osdmap e18: 3 total, 2 up, 3 in 2024-09-17T10:42:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:12.235 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:12 smithi006 ceph-mon[23740]: purged_snaps scrub starts 2024-09-17T10:42:12.235 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:12 smithi006 ceph-mon[23740]: purged_snaps scrub ok 2024-09-17T10:42:12.235 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:12.235 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:12 smithi006 ceph-mon[23740]: osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728] boot 2024-09-17T10:42:12.236 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:12 smithi006 ceph-mon[23740]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T10:42:12.236 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:12 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:12.240 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:42:12.259 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi006:vg_nvme/lv_1 2024-09-17T10:42:12.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:12 smithi149 ceph-mon[26044]: purged_snaps scrub starts 2024-09-17T10:42:12.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:12 smithi149 ceph-mon[26044]: purged_snaps scrub ok 2024-09-17T10:42:12.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:12 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:12.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:12 smithi149 ceph-mon[26044]: osd.2 [v2:172.21.15.6:6818/969866728,v1:172.21.15.6:6819/969866728] boot 2024-09-17T10:42:12.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:12 smithi149 ceph-mon[26044]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T10:42:12.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:12 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:42:13.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:13 smithi006 ceph-mon[23740]: pgmap v72: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:13.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:13 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T10:42:13.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:13 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T10:42:13.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:13 smithi006 ceph-mon[23740]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T10:42:13.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:13 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T10:42:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:13 smithi149 ceph-mon[26044]: pgmap v72: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-17T10:42:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:13 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T10:42:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:13 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T10:42:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:13 smithi149 ceph-mon[26044]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T10:42:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:13 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T10:42:13.866 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:42:13 smithi006 sudo[43226]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T10:42:13.866 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:42:13 smithi006 sudo[43226]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:13.866 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:42:13 smithi006 sudo[43226]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:14.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T10:42:14.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:14 smithi006 ceph-mon[23740]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T10:42:14.201 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43252]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T10:42:14.202 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43252]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:14.202 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43252]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:14.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T10:42:14.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:14 smithi149 ceph-mon[26044]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T10:42:14.661 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43308]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T10:42:14.662 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43308]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:14.662 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43308]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:15.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:15 smithi006 ceph-mon[23740]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:15.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:15 smithi006 ceph-mon[23740]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T10:42:15.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:15 smithi006 ceph-mon[23740]: mgrmap e19: smithi006.plhvdm(active, since 2m), standbys: smithi149.ijckag 2024-09-17T10:42:15.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:15.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:42:15.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:42:15.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:15.202 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43397]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T10:42:15.202 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43397]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:15.202 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:42:14 smithi006 sudo[43397]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:15.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:15 smithi149 ceph-mon[26044]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:15.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:15 smithi149 ceph-mon[26044]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T10:42:15.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:15 smithi149 ceph-mon[26044]: mgrmap e19: smithi006.plhvdm(active, since 2m), standbys: smithi149.ijckag 2024-09-17T10:42:15.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:15.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:42:15.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:42:15.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:15.623 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:42:15 smithi006 sudo[43464]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T10:42:15.623 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:42:15 smithi006 sudo[43464]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:15.623 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:42:15 smithi006 sudo[43464]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:15.951 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:42:15 smithi006 sudo[43501]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T10:42:15.951 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:42:15 smithi006 sudo[43501]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:15.951 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:42:15 smithi006 sudo[43501]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:16.565 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:16 smithi006 sudo[43507]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T10:42:16.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:16 smithi006 sudo[43507]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:16.566 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:16 smithi006 sudo[43507]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:16.838 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:16 smithi006 sudo[43536]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T10:42:16.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:16 smithi006 sudo[43536]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:16.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:16 smithi006 sudo[43536]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:16.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:16 smithi006 ceph-mon[23740]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:42:16.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:16 smithi006 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T10:42:17.254 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 sudo[28358]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T10:42:17.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 sudo[28358]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:17.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 sudo[28358]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 sudo[28361]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T10:42:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 sudo[28361]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T10:42:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 sudo[28361]: pam_unix(sudo:session): session closed for user root 2024-09-17T10:42:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:42:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T10:42:17.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:17.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T10:42:17.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:42:17.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3206635609' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b004b6a3-05fd-487c-9dd0-3f685bd51cdc"}]: dispatch 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3206635609' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b004b6a3-05fd-487c-9dd0-3f685bd51cdc"}]': finished 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T10:42:17.929 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:17 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T10:42:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:42:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:42:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T10:42:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:42:18.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:42:18.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T10:42:18.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:18.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:18.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3206635609' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b004b6a3-05fd-487c-9dd0-3f685bd51cdc"}]: dispatch 2024-09-17T10:42:18.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3206635609' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b004b6a3-05fd-487c-9dd0-3f685bd51cdc"}]': finished 2024-09-17T10:42:18.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T10:42:18.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:17 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:19.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:18 smithi006 ceph-mon[23740]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:19.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:18 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2641459988' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:42:19.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:18 smithi149 ceph-mon[26044]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:19.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:18 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/2641459988' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:42:20.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:20 smithi006 ceph-mon[23740]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:21.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:20 smithi149 ceph-mon[26044]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:22.864 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:22 smithi006 ceph-mon[23740]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-17T10:42:23.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:22 smithi149 ceph-mon[26044]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-17T10:42:24.188 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: Detected new or changed devices on smithi006 2024-09-17T10:42:24.188 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:24.188 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi006 to 4289M 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:24.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:23 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:24.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: Detected new or changed devices on smithi006 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi006 to 4289M 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:24.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:23 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:25.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:24 smithi006 ceph-mon[23740]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-17T10:42:25.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:24 smithi149 ceph-mon[26044]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-17T10:42:26.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:25 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T10:42:26.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:25 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:26.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:25 smithi006 ceph-mon[23740]: Deploying daemon osd.3 on smithi006 2024-09-17T10:42:26.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:25 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T10:42:26.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:25 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:26.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:25 smithi149 ceph-mon[26044]: Deploying daemon osd.3 on smithi006 2024-09-17T10:42:27.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:26 smithi006 ceph-mon[23740]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-17T10:42:27.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:26 smithi149 ceph-mon[26044]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-17T10:42:29.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:28 smithi006 ceph-mon[23740]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 77 KiB/s wr, 5 op/s 2024-09-17T10:42:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:28 smithi149 ceph-mon[26044]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 77 KiB/s wr, 5 op/s 2024-09-17T10:42:31.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:31 smithi006 ceph-mon[23740]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T10:42:31.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:31.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:31.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:31.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:31 smithi149 ceph-mon[26044]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T10:42:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:33 smithi149 ceph-mon[26044]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T10:42:33.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:33 smithi006 ceph-mon[23740]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T10:42:34.067 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 3 on host 'smithi006' 2024-09-17T10:42:34.375 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:34 smithi006 ceph-mon[23740]: from='osd.3 [v2:172.21.15.6:6826/2766264617,v1:172.21.15.6:6827/2766264617]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T10:42:34.375 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:34 smithi006 ceph-mon[23740]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T10:42:34.376 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:34 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:34 smithi149 ceph-mon[26044]: from='osd.3 [v2:172.21.15.6:6826/2766264617,v1:172.21.15.6:6827/2766264617]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T10:42:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:34 smithi149 ceph-mon[26044]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T10:42:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:34 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:34.681 DEBUG:teuthology.orchestra.run.smithi006:osd.3> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.3.service 2024-09-17T10:42:34.684 INFO:tasks.cephadm:Deploying osd.4 on smithi149 with /dev/vg_nvme/lv_4... 2024-09-17T10:42:34.684 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T10:42:35.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:35 smithi006 ceph-mon[23740]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:35.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:35 smithi006 ceph-mon[23740]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T10:42:35.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:35 smithi006 ceph-mon[23740]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T10:42:35.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:35 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:35.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:35 smithi006 ceph-mon[23740]: from='osd.3 [v2:172.21.15.6:6826/2766264617,v1:172.21.15.6:6827/2766264617]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:42:35.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:35 smithi006 ceph-mon[23740]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:42:35.202 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:42:35 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[45191]: 2024-09-17T10:42:35.137+0000 7fb09f8a4700 -1 osd.3 0 waiting for initial osdmap 2024-09-17T10:42:35.202 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:42:35 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[45191]: 2024-09-17T10:42:35.142+0000 7fb09a23b700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:42:35.265 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:35 smithi149 ceph-mon[26044]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:35.265 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:35 smithi149 ceph-mon[26044]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T10:42:35.265 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:35 smithi149 ceph-mon[26044]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T10:42:35.265 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:35 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:35.265 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:35 smithi149 ceph-mon[26044]: from='osd.3 [v2:172.21.15.6:6826/2766264617,v1:172.21.15.6:6827/2766264617]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:42:35.265 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:35 smithi149 ceph-mon[26044]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-09-17T10:42:36.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:36 smithi006 ceph-mon[23740]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-09-17T10:42:36.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:36 smithi006 ceph-mon[23740]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T10:42:36.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:36 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:36.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:36 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:36.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:36 smithi149 ceph-mon[26044]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-09-17T10:42:36.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:36 smithi149 ceph-mon[26044]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T10:42:36.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:36 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:36.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:36 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:37.188 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:42:37.207 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi149:vg_nvme/lv_4 2024-09-17T10:42:37.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:37 smithi006 ceph-mon[23740]: purged_snaps scrub starts 2024-09-17T10:42:37.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:37 smithi006 ceph-mon[23740]: purged_snaps scrub ok 2024-09-17T10:42:37.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:37 smithi006 ceph-mon[23740]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:37.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:37 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:37.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:37 smithi006 ceph-mon[23740]: osd.3 [v2:172.21.15.6:6826/2766264617,v1:172.21.15.6:6827/2766264617] boot 2024-09-17T10:42:37.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:37 smithi006 ceph-mon[23740]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T10:42:37.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:37 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:37.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:37 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:37 smithi149 ceph-mon[26044]: purged_snaps scrub starts 2024-09-17T10:42:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:37 smithi149 ceph-mon[26044]: purged_snaps scrub ok 2024-09-17T10:42:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:37 smithi149 ceph-mon[26044]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T10:42:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:37 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:37 smithi149 ceph-mon[26044]: osd.3 [v2:172.21.15.6:6826/2766264617,v1:172.21.15.6:6827/2766264617] boot 2024-09-17T10:42:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:37 smithi149 ceph-mon[26044]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T10:42:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:37 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:42:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:37 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:38.408 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:38 smithi149 ceph-mon[26044]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T10:42:38.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:38 smithi006 ceph-mon[23740]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T10:42:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:39 smithi149 ceph-mon[26044]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:39 smithi149 ceph-mon[26044]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:42:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:39 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:42:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:39 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:42:39.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:39 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:39.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:39 smithi006 ceph-mon[23740]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:39.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:39 smithi006 ceph-mon[23740]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:42:39.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:39 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:42:39.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:39 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:42:39.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:39 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:40.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:40 smithi006 ceph-mon[23740]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:40.419 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:40 smithi149 ceph-mon[26044]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:41.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi006 to 3217M 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:41.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:41.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/42088060' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aecefec1-82ec-4259-aed3-8d4d06ee5ce2"}]: dispatch 2024-09-17T10:42:41.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aecefec1-82ec-4259-aed3-8d4d06ee5ce2"}]: dispatch 2024-09-17T10:42:41.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aecefec1-82ec-4259-aed3-8d4d06ee5ce2"}]': finished 2024-09-17T10:42:41.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T10:42:41.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:41.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:41.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi006 to 3217M 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/42088060' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aecefec1-82ec-4259-aed3-8d4d06ee5ce2"}]: dispatch 2024-09-17T10:42:41.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aecefec1-82ec-4259-aed3-8d4d06ee5ce2"}]: dispatch 2024-09-17T10:42:41.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aecefec1-82ec-4259-aed3-8d4d06ee5ce2"}]': finished 2024-09-17T10:42:41.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T10:42:41.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:42.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:42 smithi006 ceph-mon[23740]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:42.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:42 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/3875392802' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:42:42.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:42 smithi149 ceph-mon[26044]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:42.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:42 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/3875392802' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:42:44.875 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:44 smithi149 ceph-mon[26044]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:44.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:44 smithi006 ceph-mon[23740]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:46 smithi149 ceph-mon[26044]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:46.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:46 smithi006 ceph-mon[23740]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T10:42:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:47 smithi149 ceph-mon[26044]: Deploying daemon osd.4 on smithi149 2024-09-17T10:42:48.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T10:42:48.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:48.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:47 smithi006 ceph-mon[23740]: Deploying daemon osd.4 on smithi149 2024-09-17T10:42:49.133 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:48 smithi149 ceph-mon[26044]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:49.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:48 smithi006 ceph-mon[23740]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:50.887 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:50 smithi149 ceph-mon[26044]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:51.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:50 smithi006 ceph-mon[23740]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:52.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:52 smithi006 ceph-mon[23740]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:52.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:52.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:52.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:52.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:52 smithi149 ceph-mon[26044]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:42:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:42:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:42:54.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:54 smithi149 ceph-mon[26044]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:54.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:54 smithi006 ceph-mon[23740]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:55.117 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 4 on host 'smithi149' 2024-09-17T10:42:55.753 DEBUG:teuthology.orchestra.run.smithi149:osd.4> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.4.service 2024-09-17T10:42:55.756 INFO:tasks.cephadm:Deploying osd.5 on smithi149 with /dev/vg_nvme/lv_3... 2024-09-17T10:42:55.756 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T10:42:55.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:55 smithi006 ceph-mon[23740]: from='osd.4 [v2:172.21.15.149:6800/761527016,v1:172.21.15.149:6801/761527016]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T10:42:55.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:55 smithi006 ceph-mon[23740]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T10:42:55.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:55 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:55 smithi149 ceph-mon[26044]: from='osd.4 [v2:172.21.15.149:6800/761527016,v1:172.21.15.149:6801/761527016]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T10:42:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:55 smithi149 ceph-mon[26044]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T10:42:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:55 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:56 smithi006 ceph-mon[23740]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:56 smithi006 ceph-mon[23740]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T10:42:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:56 smithi006 ceph-mon[23740]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T10:42:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:56 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:56.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:56 smithi006 ceph-mon[23740]: from='osd.4 [v2:172.21.15.149:6800/761527016,v1:172.21.15.149:6801/761527016]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:42:56.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:56 smithi006 ceph-mon[23740]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:42:56.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:56 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:56.956 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[30273]: 2024-09-17T10:42:56.696+0000 7f1411be9700 -1 osd.4 0 waiting for initial osdmap 2024-09-17T10:42:56.956 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[30273]: 2024-09-17T10:42:56.700+0000 7f140d582700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:42:56.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-mon[26044]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-mon[26044]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T10:42:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-mon[26044]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T10:42:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-mon[26044]: from='osd.4 [v2:172.21.15.149:6800/761527016,v1:172.21.15.149:6801/761527016]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:42:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-mon[26044]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:42:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:56 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:42:57.876 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:57 smithi149 ceph-mon[26044]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-09-17T10:42:57.876 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:57 smithi149 ceph-mon[26044]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T10:42:57.876 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:57.877 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:57.877 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:57 smithi149 ceph-mon[26044]: osd.4 [v2:172.21.15.149:6800/761527016,v1:172.21.15.149:6801/761527016] boot 2024-09-17T10:42:57.877 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:57 smithi149 ceph-mon[26044]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T10:42:57.877 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:57.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:57 smithi006 ceph-mon[23740]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-09-17T10:42:57.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:57 smithi006 ceph-mon[23740]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T10:42:57.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:57.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:57.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:57 smithi006 ceph-mon[23740]: osd.4 [v2:172.21.15.149:6800/761527016,v1:172.21.15.149:6801/761527016] boot 2024-09-17T10:42:57.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:57 smithi006 ceph-mon[23740]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T10:42:57.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:42:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:58 smithi149 ceph-mon[26044]: purged_snaps scrub starts 2024-09-17T10:42:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:58 smithi149 ceph-mon[26044]: purged_snaps scrub ok 2024-09-17T10:42:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:58 smithi149 ceph-mon[26044]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:58 smithi149 ceph-mon[26044]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T10:42:59.079 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:42:59.101 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi149:vg_nvme/lv_3 2024-09-17T10:42:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:58 smithi006 ceph-mon[23740]: purged_snaps scrub starts 2024-09-17T10:42:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:58 smithi006 ceph-mon[23740]: purged_snaps scrub ok 2024-09-17T10:42:59.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:58 smithi006 ceph-mon[23740]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T10:42:59.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:58 smithi006 ceph-mon[23740]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T10:43:00.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:42:59 smithi006 ceph-mon[23740]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T10:43:00.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:42:59 smithi149 ceph-mon[26044]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T10:43:01.166 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:00 smithi149 ceph-mon[26044]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:01.166 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:00 smithi149 ceph-mon[26044]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T10:43:01.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:00 smithi006 ceph-mon[23740]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:01.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:00 smithi006 ceph-mon[23740]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T10:43:02.033 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T10:43:02.033 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: Detected new or changed devices on smithi149 2024-09-17T10:43:02.033 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:02.033 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi149 to 4663M 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:43:02.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:43:02.035 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:43:02.035 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:02.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: Detected new or changed devices on smithi149 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi149 to 4663M 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:43:02.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:43:02.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:43:02.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:43:02.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:03.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:03 smithi006 ceph-mon[23740]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-17T10:43:03.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:03 smithi006 ceph-mon[23740]: from='client.24161 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:43:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:03 smithi149 ceph-mon[26044]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-17T10:43:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:03 smithi149 ceph-mon[26044]: from='client.24161 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:43:04.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:04 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/18102410' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12c7592b-af31-44a4-b872-26fb12b6932b"}]: dispatch 2024-09-17T10:43:04.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:04 smithi006 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12c7592b-af31-44a4-b872-26fb12b6932b"}]: dispatch 2024-09-17T10:43:04.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:04 smithi006 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12c7592b-af31-44a4-b872-26fb12b6932b"}]': finished 2024-09-17T10:43:04.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:04 smithi006 ceph-mon[23740]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T10:43:04.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:04 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:04 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/18102410' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12c7592b-af31-44a4-b872-26fb12b6932b"}]: dispatch 2024-09-17T10:43:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:04 smithi149 ceph-mon[26044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12c7592b-af31-44a4-b872-26fb12b6932b"}]: dispatch 2024-09-17T10:43:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:04 smithi149 ceph-mon[26044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12c7592b-af31-44a4-b872-26fb12b6932b"}]': finished 2024-09-17T10:43:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:04 smithi149 ceph-mon[26044]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T10:43:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:04 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:05.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:05 smithi149 ceph-mon[26044]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-09-17T10:43:05.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:05 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/2180705946' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:43:05.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:05 smithi006 ceph-mon[23740]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-09-17T10:43:05.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:05 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/2180705946' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:43:07.429 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:07 smithi149 ceph-mon[26044]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T10:43:07.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:07 smithi006 ceph-mon[23740]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T10:43:09.272 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:09 smithi149 ceph-mon[26044]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-17T10:43:09.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:09 smithi006 ceph-mon[23740]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-17T10:43:10.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T10:43:10.255 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:10.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T10:43:10.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:11.347 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:11 smithi149 ceph-mon[26044]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:11.348 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:11 smithi149 ceph-mon[26044]: Deploying daemon osd.5 on smithi149 2024-09-17T10:43:11.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:11 smithi006 ceph-mon[23740]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:11.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:11 smithi006 ceph-mon[23740]: Deploying daemon osd.5 on smithi149 2024-09-17T10:43:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:13 smithi149 ceph-mon[26044]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:13.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:13 smithi006 ceph-mon[23740]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:15.429 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:15 smithi149 ceph-mon[26044]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:15.429 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:15.429 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:15.429 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:15.429 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:15 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:15.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:15 smithi006 ceph-mon[23740]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:15.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:15.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:15.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:15.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:15 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:17 smithi149 ceph-mon[26044]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:17.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:17 smithi006 ceph-mon[23740]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:17.805 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 5 on host 'smithi149' 2024-09-17T10:43:18.377 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:18 smithi149 ceph-mon[26044]: from='osd.5 [v2:172.21.15.149:6808/1397443451,v1:172.21.15.149:6809/1397443451]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T10:43:18.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:18 smithi149 ceph-mon[26044]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T10:43:18.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:18 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:18.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:18 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:18.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:18 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:18.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:18 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:18.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:18 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:18.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:18 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:18.404 DEBUG:teuthology.orchestra.run.smithi149:osd.5> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.5.service 2024-09-17T10:43:18.406 INFO:tasks.cephadm:Deploying osd.6 on smithi149 with /dev/vg_nvme/lv_2... 2024-09-17T10:43:18.406 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T10:43:18.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:18 smithi006 ceph-mon[23740]: from='osd.5 [v2:172.21.15.149:6808/1397443451,v1:172.21.15.149:6809/1397443451]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T10:43:18.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:18 smithi006 ceph-mon[23740]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T10:43:18.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:18 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:18.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:18 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:18.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:18 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:18.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:18 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:18.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:18 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:18.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:18 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:19.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:19 smithi149 ceph-mon[26044]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:19.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:19 smithi149 ceph-mon[26044]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T10:43:19.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:19 smithi149 ceph-mon[26044]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T10:43:19.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:19 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:19.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:19 smithi149 ceph-mon[26044]: from='osd.5 [v2:172.21.15.149:6808/1397443451,v1:172.21.15.149:6809/1397443451]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:43:19.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:19 smithi149 ceph-mon[26044]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:43:19.335 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:43:19 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[33217]: 2024-09-17T10:43:19.140+0000 7f610e6cd700 -1 osd.5 0 waiting for initial osdmap 2024-09-17T10:43:19.335 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:43:19 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[33217]: 2024-09-17T10:43:19.148+0000 7f6109064700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:43:19.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:19 smithi006 ceph-mon[23740]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:19.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:19 smithi006 ceph-mon[23740]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T10:43:19.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:19 smithi006 ceph-mon[23740]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T10:43:19.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:19 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:19.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:19 smithi006 ceph-mon[23740]: from='osd.5 [v2:172.21.15.149:6808/1397443451,v1:172.21.15.149:6809/1397443451]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:43:19.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:19 smithi006 ceph-mon[23740]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:43:20.293 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:20 smithi149 ceph-mon[26044]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-09-17T10:43:20.293 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:20 smithi149 ceph-mon[26044]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T10:43:20.293 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:20 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:20.293 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:20 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:20.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:20 smithi006 ceph-mon[23740]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-09-17T10:43:20.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:20 smithi006 ceph-mon[23740]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T10:43:20.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:20 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:20.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:20 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:21.288 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:21 smithi149 ceph-mon[26044]: purged_snaps scrub starts 2024-09-17T10:43:21.288 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:21 smithi149 ceph-mon[26044]: purged_snaps scrub ok 2024-09-17T10:43:21.288 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:21 smithi149 ceph-mon[26044]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:21.289 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:21 smithi149 ceph-mon[26044]: osd.5 [v2:172.21.15.149:6808/1397443451,v1:172.21.15.149:6809/1397443451] boot 2024-09-17T10:43:21.289 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:21 smithi149 ceph-mon[26044]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T10:43:21.289 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:21 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:21.289 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:21 smithi149 ceph-mon[26044]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T10:43:21.293 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:43:21.313 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi149:vg_nvme/lv_2 2024-09-17T10:43:21.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:21 smithi006 ceph-mon[23740]: purged_snaps scrub starts 2024-09-17T10:43:21.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:21 smithi006 ceph-mon[23740]: purged_snaps scrub ok 2024-09-17T10:43:21.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:21 smithi006 ceph-mon[23740]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T10:43:21.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:21 smithi006 ceph-mon[23740]: osd.5 [v2:172.21.15.149:6808/1397443451,v1:172.21.15.149:6809/1397443451] boot 2024-09-17T10:43:21.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:21 smithi006 ceph-mon[23740]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T10:43:21.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:21 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:43:21.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:21 smithi006 ceph-mon[23740]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T10:43:23.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:23 smithi006 ceph-mon[23740]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:23.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:23 smithi006 ceph-mon[23740]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T10:43:23.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:23 smithi006 ceph-mon[23740]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T10:43:23.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:23 smithi149 ceph-mon[26044]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:23.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:23 smithi149 ceph-mon[26044]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T10:43:23.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:23 smithi149 ceph-mon[26044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T10:43:24.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T10:43:24.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:24.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: Detected new or changed devices on smithi149 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi149 to 2331M 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:24.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:24 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:24.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T10:43:24.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:24.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:43:24.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: Detected new or changed devices on smithi149 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi149 to 2331M 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:24.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:24 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:25.197 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:25 smithi149 ceph-mon[26044]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T10:43:25.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:25 smithi006 ceph-mon[23740]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T10:43:26.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:26 smithi006 ceph-mon[23740]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-09-17T10:43:26.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:26 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/3011940779' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3863943d-9cdf-47b9-bf2a-0641a445e2a1"}]: dispatch 2024-09-17T10:43:26.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:26 smithi006 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3863943d-9cdf-47b9-bf2a-0641a445e2a1"}]: dispatch 2024-09-17T10:43:26.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:26 smithi006 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3863943d-9cdf-47b9-bf2a-0641a445e2a1"}]': finished 2024-09-17T10:43:26.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:26 smithi006 ceph-mon[23740]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T10:43:26.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:26 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:26 smithi149 ceph-mon[26044]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-09-17T10:43:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:26 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/3011940779' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3863943d-9cdf-47b9-bf2a-0641a445e2a1"}]: dispatch 2024-09-17T10:43:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:26 smithi149 ceph-mon[26044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3863943d-9cdf-47b9-bf2a-0641a445e2a1"}]: dispatch 2024-09-17T10:43:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:26 smithi149 ceph-mon[26044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3863943d-9cdf-47b9-bf2a-0641a445e2a1"}]': finished 2024-09-17T10:43:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:26 smithi149 ceph-mon[26044]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T10:43:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:26 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:27.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:27 smithi006 ceph-mon[23740]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T10:43:27.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:27 smithi006 ceph-mon[23740]: Cluster is now healthy 2024-09-17T10:43:27.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:27 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/1054997764' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:43:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:27 smithi149 ceph-mon[26044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T10:43:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:27 smithi149 ceph-mon[26044]: Cluster is now healthy 2024-09-17T10:43:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:27 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/1054997764' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:43:28.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:28 smithi006 ceph-mon[23740]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-17T10:43:28.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:28 smithi149 ceph-mon[26044]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-17T10:43:30.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:30 smithi149 ceph-mon[26044]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-17T10:43:30.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:30 smithi006 ceph-mon[23740]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-17T10:43:33.113 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:32 smithi149 ceph-mon[26044]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T10:43:33.113 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T10:43:33.113 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:33.113 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:32 smithi149 ceph-mon[26044]: Deploying daemon osd.6 on smithi149 2024-09-17T10:43:33.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:32 smithi006 ceph-mon[23740]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T10:43:33.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T10:43:33.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:33.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:32 smithi006 ceph-mon[23740]: Deploying daemon osd.6 on smithi149 2024-09-17T10:43:35.120 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:34 smithi149 ceph-mon[26044]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-17T10:43:35.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:34 smithi006 ceph-mon[23740]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-17T10:43:36.881 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:36 smithi149 ceph-mon[26044]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:36 smithi006 ceph-mon[23740]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:38.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:37 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:38.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:37 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:38.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:37 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:38.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:37 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:38.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:37 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:38.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:37 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:38.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:37 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:38.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:37 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:39.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:38 smithi006 ceph-mon[23740]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:38 smithi149 ceph-mon[26044]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:40.340 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 6 on host 'smithi149' 2024-09-17T10:43:41.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:40 smithi006 ceph-mon[23740]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:41.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:40 smithi006 ceph-mon[23740]: from='osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T10:43:41.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:40 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:40 smithi149 ceph-mon[26044]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:40 smithi149 ceph-mon[26044]: from='osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T10:43:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:40 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:41.278 DEBUG:teuthology.orchestra.run.smithi149:osd.6> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.6.service 2024-09-17T10:43:41.281 INFO:tasks.cephadm:Deploying osd.7 on smithi149 with /dev/vg_nvme/lv_1... 2024-09-17T10:43:41.281 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T10:43:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:41 smithi006 ceph-mon[23740]: from='osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T10:43:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:41 smithi006 ceph-mon[23740]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T10:43:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:41 smithi006 ceph-mon[23740]: from='osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:43:42.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:41 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:41 smithi149 ceph-mon[26044]: from='osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T10:43:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:41 smithi149 ceph-mon[26044]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T10:43:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:41 smithi149 ceph-mon[26044]: from='osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:43:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:41 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:42.206 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:43:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[36188]: 2024-09-17T10:43:41.947+0000 7f36aafad700 -1 osd.6 0 waiting for initial osdmap 2024-09-17T10:43:42.207 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:43:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[36188]: 2024-09-17T10:43:41.954+0000 7f36a4141700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:43:43.063 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:42 smithi149 ceph-mon[26044]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:43.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:42 smithi149 ceph-mon[26044]: from='osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-09-17T10:43:43.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:42 smithi149 ceph-mon[26044]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T10:43:43.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:42 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:43.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:42 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:43.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:42 smithi006 ceph-mon[23740]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T10:43:43.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:42 smithi006 ceph-mon[23740]: from='osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-09-17T10:43:43.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:42 smithi006 ceph-mon[23740]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T10:43:43.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:42 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:43.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:42 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:44.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:43 smithi149 ceph-mon[26044]: purged_snaps scrub starts 2024-09-17T10:43:44.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:43 smithi149 ceph-mon[26044]: purged_snaps scrub ok 2024-09-17T10:43:44.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:43 smithi149 ceph-mon[26044]: osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945] boot 2024-09-17T10:43:44.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:43 smithi149 ceph-mon[26044]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T10:43:44.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:43 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:44.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:43 smithi006 ceph-mon[23740]: purged_snaps scrub starts 2024-09-17T10:43:44.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:43 smithi006 ceph-mon[23740]: purged_snaps scrub ok 2024-09-17T10:43:44.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:43 smithi006 ceph-mon[23740]: osd.6 [v2:172.21.15.149:6816/3219489945,v1:172.21.15.149:6817/3219489945] boot 2024-09-17T10:43:44.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:43 smithi006 ceph-mon[23740]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T10:43:44.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:43 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:43:44.834 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-09-17T10:43:44.853 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi149:vg_nvme/lv_1 2024-09-17T10:43:45.106 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:44 smithi149 ceph-mon[26044]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:43:45.107 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:44 smithi149 ceph-mon[26044]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T10:43:45.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:44 smithi006 ceph-mon[23740]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:43:45.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:44 smithi006 ceph-mon[23740]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T10:43:46.067 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:45 smithi149 ceph-mon[26044]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T10:43:46.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:45 smithi006 ceph-mon[23740]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T10:43:47.077 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:43:47.078 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T10:43:47.078 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:47.078 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:47.078 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:47.078 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:47.078 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:47.079 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:47.079 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:47.079 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:47.079 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:47.290 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:43:47.290 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T10:43:47.290 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:47.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:47.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:47.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:43:47.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:47.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:47.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:43:47.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:47.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:43:48.130 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:47 smithi149 ceph-mon[26044]: Detected new or changed devices on smithi149 2024-09-17T10:43:48.131 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:47 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi149 to 1554M 2024-09-17T10:43:48.131 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:47 smithi149 ceph-mon[26044]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T10:43:48.131 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:48.131 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:43:48.131 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:43:48.131 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:48.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:47 smithi006 ceph-mon[23740]: Detected new or changed devices on smithi149 2024-09-17T10:43:48.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:47 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi149 to 1554M 2024-09-17T10:43:48.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:47 smithi006 ceph-mon[23740]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T10:43:48.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:43:48.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T10:43:48.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T10:43:48.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:49.185 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:49 smithi149 ceph-mon[26044]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:43:49.186 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:49 smithi149 ceph-mon[26044]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:43:49.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:49 smithi006 ceph-mon[23740]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:43:49.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:49 smithi006 ceph-mon[23740]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:43:50.368 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:50 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/3819557922' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "979b6e41-161a-45a2-bc8b-1dd980e78bf6"}]: dispatch 2024-09-17T10:43:50.369 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:50 smithi149 ceph-mon[26044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "979b6e41-161a-45a2-bc8b-1dd980e78bf6"}]: dispatch 2024-09-17T10:43:50.369 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:50 smithi149 ceph-mon[26044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "979b6e41-161a-45a2-bc8b-1dd980e78bf6"}]': finished 2024-09-17T10:43:50.369 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:50 smithi149 ceph-mon[26044]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T10:43:50.369 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:50 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:43:50.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:50 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/3819557922' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "979b6e41-161a-45a2-bc8b-1dd980e78bf6"}]: dispatch 2024-09-17T10:43:50.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:50 smithi006 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "979b6e41-161a-45a2-bc8b-1dd980e78bf6"}]: dispatch 2024-09-17T10:43:50.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:50 smithi006 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "979b6e41-161a-45a2-bc8b-1dd980e78bf6"}]': finished 2024-09-17T10:43:50.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:50 smithi006 ceph-mon[23740]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T10:43:50.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:50 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:43:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:51 smithi149 ceph-mon[26044]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-09-17T10:43:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:51 smithi149 ceph-mon[26044]: from='client.? 172.21.15.149:0/1958957997' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:43:51.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:51 smithi006 ceph-mon[23740]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-09-17T10:43:51.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:51 smithi006 ceph-mon[23740]: from='client.? 172.21.15.149:0/1958957997' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T10:43:53.319 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:53 smithi149 ceph-mon[26044]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T10:43:53.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:53 smithi006 ceph-mon[23740]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T10:43:55.354 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:55 smithi149 ceph-mon[26044]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-17T10:43:55.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:55 smithi006 ceph-mon[23740]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-17T10:43:56.441 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:56 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T10:43:56.441 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:56 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:56.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:56 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T10:43:56.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:56 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:43:57.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:57 smithi006 ceph-mon[23740]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-17T10:43:57.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:57 smithi006 ceph-mon[23740]: Deploying daemon osd.7 on smithi149 2024-09-17T10:43:57.454 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:57 smithi149 ceph-mon[26044]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-17T10:43:57.454 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:57 smithi149 ceph-mon[26044]: Deploying daemon osd.7 on smithi149 2024-09-17T10:43:59.314 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:43:59 smithi149 ceph-mon[26044]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T10:43:59.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:43:59 smithi006 ceph-mon[23740]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T10:44:01.406 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:01 smithi149 ceph-mon[26044]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:44:01.407 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:01.407 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:44:01.407 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:44:01.407 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:44:01.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:01 smithi006 ceph-mon[23740]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:44:01.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:01.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:44:01.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:44:01.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:44:02.403 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:44:02.403 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:44:02.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:44:02.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:44:03.170 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:03 smithi149 ceph-mon[26044]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:44:03.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:03 smithi006 ceph-mon[23740]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:44:03.747 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 7 on host 'smithi149' 2024-09-17T10:44:04.286 DEBUG:teuthology.orchestra.run.smithi149:osd.7> sudo journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.7.service 2024-09-17T10:44:04.289 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-17T10:44:04.289 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-17T10:44:04.825 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:04 smithi149 ceph-mon[26044]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:44:04.825 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:04 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:04.825 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:04 smithi149 ceph-mon[26044]: from='osd.7 [v2:172.21.15.149:6824/2180530477,v1:172.21.15.149:6825/2180530477]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T10:44:04.825 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:04 smithi149 ceph-mon[26044]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T10:44:04.909 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:04 smithi006 ceph-mon[23740]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:44:04.909 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:04 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:04.909 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:04 smithi006 ceph-mon[23740]: from='osd.7 [v2:172.21.15.149:6824/2180530477,v1:172.21.15.149:6825/2180530477]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T10:44:04.909 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:04 smithi006 ceph-mon[23740]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T10:44:05.829 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:05 smithi149 ceph-mon[26044]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T10:44:05.829 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:05 smithi149 ceph-mon[26044]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T10:44:05.829 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:05 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:44:05.830 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:05 smithi149 ceph-mon[26044]: from='osd.7 [v2:172.21.15.149:6824/2180530477,v1:172.21.15.149:6825/2180530477]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:44:05.830 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:05 smithi149 ceph-mon[26044]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:44:05.830 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:05 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:05.830 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:44:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[39248]: 2024-09-17T10:44:05.767+0000 7fd1fd0f8700 -1 osd.7 0 waiting for initial osdmap 2024-09-17T10:44:05.831 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:44:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[39248]: 2024-09-17T10:44:05.775+0000 7fd1f728e700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:44:05.902 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:05 smithi006 ceph-mon[23740]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T10:44:05.902 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:05 smithi006 ceph-mon[23740]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T10:44:05.902 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:05 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:44:05.902 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:05 smithi006 ceph-mon[23740]: from='osd.7 [v2:172.21.15.149:6824/2180530477,v1:172.21.15.149:6825/2180530477]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:44:05.902 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:05 smithi006 ceph-mon[23740]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-09-17T10:44:05.903 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:05 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:06.453 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:44:07.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:44:07.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-09-17T10:44:07.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T10:44:07.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:44:07.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:44:07.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/435184391' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T10:44:07.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: osd.7 [v2:172.21.15.149:6824/2180530477,v1:172.21.15.149:6825/2180530477] boot 2024-09-17T10:44:07.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T10:44:07.062 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:06 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/435184391' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: osd.7 [v2:172.21.15.149:6824/2180530477,v1:172.21.15.149:6825/2180530477] boot 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T10:44:07.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:06 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:44:07.477 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1726569822,"num_in_osds":8,"osd_in_since":1726569829,"num_remapped_pgs":0} 2024-09-17T10:44:08.047 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:07 smithi149 ceph-mon[26044]: purged_snaps scrub starts 2024-09-17T10:44:08.047 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:07 smithi149 ceph-mon[26044]: purged_snaps scrub ok 2024-09-17T10:44:08.047 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:07 smithi149 ceph-mon[26044]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T10:44:08.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:07 smithi006 ceph-mon[23740]: purged_snaps scrub starts 2024-09-17T10:44:08.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:07 smithi006 ceph-mon[23740]: purged_snaps scrub ok 2024-09-17T10:44:08.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:07 smithi006 ceph-mon[23740]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T10:44:08.480 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-17T10:44:09.055 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:09.055 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T10:44:09.055 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: Detected new or changed devices on smithi149 2024-09-17T10:44:09.055 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:09.055 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi149 to 1165M 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:44:09.056 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: Detected new or changed devices on smithi149 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi149 to 1165M 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:44:09.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:44:09.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:44:10.147 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:09 smithi006 ceph-mon[23740]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T10:44:10.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:09 smithi149 ceph-mon[26044]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T10:44:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:44:11.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:10 smithi006 ceph-mon[23740]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:11.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:10 smithi006 ceph-mon[23740]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T10:44:11.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:10 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3133053214' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T10:44:11.111 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1726569846,"num_in_osds":8,"osd_in_since":1726569829,"num_remapped_pgs":0} 2024-09-17T10:44:11.111 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T10:44:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:10 smithi149 ceph-mon[26044]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:10 smithi149 ceph-mon[26044]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T10:44:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:10 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3133053214' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T10:44:11.333 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:13.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:12 smithi006 ceph-mon[23740]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:12 smithi149 ceph-mon[26044]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:13.266 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:44:13.266 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":59,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","created":"2024-09-17T10:38:16.982968+0000","modified":"2024-09-17T10:44:09.808633+0000","last_up_change":"2024-09-17T10:44:06.753584+0000","last_in_change":"2024-09-17T10:43:49.875759+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T10:42:12.008115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"2fbbec5b-85fb-4bd7-82ef-adc149c87960","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6803","nonce":3722513843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6805","nonce":3722513843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6809","nonce":3722513843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6807","nonce":3722513843}]},"public_addr":"172.21.15.6:6803/3722513843","cluster_addr":"172.21.15.6:6805/3722513843","heartbeat_back_addr":"172.21.15.6:6809/3722513843","heartbeat_front_addr":"172.21.15.6:6807/3722513843","state":["exists","up"]},{"osd":1,"uuid":"927193ac-f68c-463b-b254-a0096ab97bbb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6811","nonce":3766560025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6813","nonce":3766560025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6817","nonce":3766560025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6815","nonce":3766560025}]},"public_addr":"172.21.15.6:6811/3766560025","cluster_addr":"172.21.15.6:6813/3766560025","heartbeat_back_addr":"172.21.15.6:6817/3766560025","heartbeat_front_addr":"172.21.15.6:6815/3766560025","state":["exists","up"]},{"osd":2,"uuid":"ef897e3f-bf8c-4430-a3a6-ead5208c6a51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6819","nonce":969866728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6821","nonce":969866728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6825","nonce":969866728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6823","nonce":969866728}]},"public_addr":"172.21.15.6:6819/969866728","cluster_addr":"172.21.15.6:6821/969866728","heartbeat_back_addr":"172.21.15.6:6825/969866728","heartbeat_front_addr":"172.21.15.6:6823/969866728","state":["exists","up"]},{"osd":3,"uuid":"b004b6a3-05fd-487c-9dd0-3f685bd51cdc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6827","nonce":2766264617}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6829","nonce":2766264617}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6833","nonce":2766264617}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6831","nonce":2766264617}]},"public_addr":"172.21.15.6:6827/2766264617","cluster_addr":"172.21.15.6:6829/2766264617","heartbeat_back_addr":"172.21.15.6:6833/2766264617","heartbeat_front_addr":"172.21.15.6:6831/2766264617","state":["exists","up"]},{"osd":4,"uuid":"aecefec1-82ec-4259-aed3-8d4d06ee5ce2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6801","nonce":761527016}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6803","nonce":761527016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6807","nonce":761527016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6805","nonce":761527016}]},"public_addr":"172.21.15.149:6801/761527016","cluster_addr":"172.21.15.149:6803/761527016","heartbeat_back_addr":"172.21.15.149:6807/761527016","heartbeat_front_addr":"172.21.15.149:6805/761527016","state":["exists","up"]},{"osd":5,"uuid":"12c7592b-af31-44a4-b872-26fb12b6932b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6809","nonce":1397443451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6811","nonce":1397443451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6815","nonce":1397443451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6813","nonce":1397443451}]},"public_addr":"172.21.15.149:6809/1397443451","cluster_addr":"172.21.15.149:6811/1397443451","heartbeat_back_addr":"172.21.15.149:6815/1397443451","heartbeat_front_addr":"172.21.15.149:6813/1397443451","state":["exists","up"]},{"osd":6,"uuid":"3863943d-9cdf-47b9-bf2a-0641a445e2a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6817","nonce":3219489945}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6819","nonce":3219489945}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6823","nonce":3219489945}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6821","nonce":3219489945}]},"public_addr":"172.21.15.149:6817/3219489945","cluster_addr":"172.21.15.149:6819/3219489945","heartbeat_back_addr":"172.21.15.149:6823/3219489945","heartbeat_front_addr":"172.21.15.149:6821/3219489945","state":["exists","up"]},{"osd":7,"uuid":"979b6e41-161a-45a2-bc8b-1dd980e78bf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6825","nonce":2180530477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6827","nonce":2180530477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6831","nonce":2180530477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6829","nonce":2180530477}]},"public_addr":"172.21.15.149:6825/2180530477","cluster_addr":"172.21.15.149:6827/2180530477","heartbeat_back_addr":"172.21.15.149:6831/2180530477","heartbeat_front_addr":"172.21.15.149:6829/2180530477","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:41:11.883494+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:41:42.408147+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:08.990744+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:34.229094+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:55.844856+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:43:18.190182+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:43:41.095017+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:44:04.979017+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/271712424":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/3791438495":"2024-09-18T10:40:01.448980+0000","172.21.15.6:6801/411654524":"2024-09-18T10:40:01.448980+0000","172.21.15.6:6800/411654524":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/194568551":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/1873228168":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6800/2106963250":"2024-09-18T10:39:12.608826+0000","172.21.15.6:0/1062781985":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/2550476919":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/1670227312":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6801/2106963250":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6800/3033656982":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/2729884814":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/3074268711":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6801/3033656982":"2024-09-18T10:38:42.872777+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T10:44:13.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:13 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/736075098' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T10:44:13.993 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-17T10:42:12.008115+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-17T10:44:13.993 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-17T10:44:14.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:13 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/736075098' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T10:44:14.214 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:14.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:14 smithi006 ceph-mon[23740]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T10:44:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:14 smithi149 ceph-mon[26044]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T10:44:16.157 INFO:teuthology.orchestra.run.smithi006.stdout:pg_num: 1 2024-09-17T10:44:16.914 INFO:tasks.cephadm:Setting up client nodes... 2024-09-17T10:44:16.914 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-17T10:44:17.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:16 smithi006 ceph-mon[23740]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-17T10:44:17.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:16 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/623124636' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T10:44:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:16 smithi149 ceph-mon[26044]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-17T10:44:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:16 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/623124636' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T10:44:19.097 INFO:teuthology.orchestra.run.smithi006.stdout:[client.0] 2024-09-17T10:44:19.098 INFO:teuthology.orchestra.run.smithi006.stdout: key = AQCDXelmX75qBRAAeZQP5cSGRI7KXLrOKk07ow== 2024-09-17T10:44:19.098 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:18 smithi006 ceph-mon[23740]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-17T10:44:19.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:18 smithi149 ceph-mon[26044]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-17T10:44:19.759 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-09-17T10:44:19.759 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-17T10:44:19.759 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-17T10:44:19.799 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-17T10:44:20.059 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:20.181 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:19 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2373443697' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T10:44:20.181 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:19 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2373443697' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T10:44:20.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:19 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/2373443697' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T10:44:20.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:19 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/2373443697' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T10:44:21.368 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:21 smithi006 ceph-mon[23740]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T10:44:21.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:21 smithi149 ceph-mon[26044]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T10:44:21.977 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:44:22.768 INFO:teuthology.orchestra.run.smithi006.stdout:[{"version":21,"timestamp":"2024-09-17T10:44:08.081744+0000","name":"","changes":[{"name":"osd/host:smithi149/osd_memory_target","previous_value":"1629947767","new_value":"1222460825"}]}] 2024-09-17T10:44:22.768 INFO:tasks.ceph_manager:config epoch is 21 2024-09-17T10:44:22.769 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-17T10:44:22.769 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-17T10:44:22.769 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-17T10:44:22.989 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:23.108 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:22 smithi006 ceph-mon[23740]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-17T10:44:23.108 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:22 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3982375397' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-17T10:44:23.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:22 smithi149 ceph-mon[26044]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-17T10:44:23.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:22 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3982375397' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-17T10:44:24.925 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:44:25.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:24 smithi006 ceph-mon[23740]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T10:44:25.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:24 smithi149 ceph-mon[26044]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T10:44:25.725 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi006.plhvdm","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6800","nonce":3145151523},{"type":"v1","addr":"172.21.15.6:6801","nonce":3145151523}]},"active_addr":"172.21.15.6:6801/3145151523","active_change":"2024-09-17T10:40:01.449275+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi149.ijckag","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.6:8443/","prometheus":"http://172.21.15.6:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":3122121985}]},{"addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":3645904492}]},{"addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":2410363225}]},{"addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":135792680}]}]}} 2024-09-17T10:44:25.729 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-17T10:44:25.729 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-17T10:44:25.729 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T10:44:25.950 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:25.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:25 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1168807470' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T10:44:26.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:25 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1168807470' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T10:44:27.092 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:26 smithi006 ceph-mon[23740]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:27.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:26 smithi149 ceph-mon[26044]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:27.901 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:44:27.901 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":59,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","created":"2024-09-17T10:38:16.982968+0000","modified":"2024-09-17T10:44:09.808633+0000","last_up_change":"2024-09-17T10:44:06.753584+0000","last_in_change":"2024-09-17T10:43:49.875759+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T10:42:12.008115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"2fbbec5b-85fb-4bd7-82ef-adc149c87960","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6803","nonce":3722513843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6805","nonce":3722513843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6809","nonce":3722513843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6807","nonce":3722513843}]},"public_addr":"172.21.15.6:6803/3722513843","cluster_addr":"172.21.15.6:6805/3722513843","heartbeat_back_addr":"172.21.15.6:6809/3722513843","heartbeat_front_addr":"172.21.15.6:6807/3722513843","state":["exists","up"]},{"osd":1,"uuid":"927193ac-f68c-463b-b254-a0096ab97bbb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6811","nonce":3766560025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6813","nonce":3766560025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6817","nonce":3766560025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6815","nonce":3766560025}]},"public_addr":"172.21.15.6:6811/3766560025","cluster_addr":"172.21.15.6:6813/3766560025","heartbeat_back_addr":"172.21.15.6:6817/3766560025","heartbeat_front_addr":"172.21.15.6:6815/3766560025","state":["exists","up"]},{"osd":2,"uuid":"ef897e3f-bf8c-4430-a3a6-ead5208c6a51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6819","nonce":969866728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6821","nonce":969866728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6825","nonce":969866728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6823","nonce":969866728}]},"public_addr":"172.21.15.6:6819/969866728","cluster_addr":"172.21.15.6:6821/969866728","heartbeat_back_addr":"172.21.15.6:6825/969866728","heartbeat_front_addr":"172.21.15.6:6823/969866728","state":["exists","up"]},{"osd":3,"uuid":"b004b6a3-05fd-487c-9dd0-3f685bd51cdc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6827","nonce":2766264617}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6829","nonce":2766264617}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6833","nonce":2766264617}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6831","nonce":2766264617}]},"public_addr":"172.21.15.6:6827/2766264617","cluster_addr":"172.21.15.6:6829/2766264617","heartbeat_back_addr":"172.21.15.6:6833/2766264617","heartbeat_front_addr":"172.21.15.6:6831/2766264617","state":["exists","up"]},{"osd":4,"uuid":"aecefec1-82ec-4259-aed3-8d4d06ee5ce2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6801","nonce":761527016}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6803","nonce":761527016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6807","nonce":761527016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6805","nonce":761527016}]},"public_addr":"172.21.15.149:6801/761527016","cluster_addr":"172.21.15.149:6803/761527016","heartbeat_back_addr":"172.21.15.149:6807/761527016","heartbeat_front_addr":"172.21.15.149:6805/761527016","state":["exists","up"]},{"osd":5,"uuid":"12c7592b-af31-44a4-b872-26fb12b6932b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6809","nonce":1397443451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6811","nonce":1397443451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6815","nonce":1397443451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6813","nonce":1397443451}]},"public_addr":"172.21.15.149:6809/1397443451","cluster_addr":"172.21.15.149:6811/1397443451","heartbeat_back_addr":"172.21.15.149:6815/1397443451","heartbeat_front_addr":"172.21.15.149:6813/1397443451","state":["exists","up"]},{"osd":6,"uuid":"3863943d-9cdf-47b9-bf2a-0641a445e2a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6817","nonce":3219489945}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6819","nonce":3219489945}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6823","nonce":3219489945}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6821","nonce":3219489945}]},"public_addr":"172.21.15.149:6817/3219489945","cluster_addr":"172.21.15.149:6819/3219489945","heartbeat_back_addr":"172.21.15.149:6823/3219489945","heartbeat_front_addr":"172.21.15.149:6821/3219489945","state":["exists","up"]},{"osd":7,"uuid":"979b6e41-161a-45a2-bc8b-1dd980e78bf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6825","nonce":2180530477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6827","nonce":2180530477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6831","nonce":2180530477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6829","nonce":2180530477}]},"public_addr":"172.21.15.149:6825/2180530477","cluster_addr":"172.21.15.149:6827/2180530477","heartbeat_back_addr":"172.21.15.149:6831/2180530477","heartbeat_front_addr":"172.21.15.149:6829/2180530477","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:41:11.883494+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:41:42.408147+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:08.990744+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:34.229094+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:55.844856+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:43:18.190182+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:43:41.095017+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:44:04.979017+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/271712424":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/3791438495":"2024-09-18T10:40:01.448980+0000","172.21.15.6:6801/411654524":"2024-09-18T10:40:01.448980+0000","172.21.15.6:6800/411654524":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/194568551":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/1873228168":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6800/2106963250":"2024-09-18T10:39:12.608826+0000","172.21.15.6:0/1062781985":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/2550476919":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/1670227312":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6801/2106963250":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6800/3033656982":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/2729884814":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/3074268711":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6801/3033656982":"2024-09-18T10:38:42.872777+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T10:44:28.597 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-17T10:44:28.598 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T10:44:28.817 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:28.937 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:28 smithi006 ceph-mon[23740]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:28.937 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:28 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3566502934' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T10:44:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:28 smithi149 ceph-mon[26044]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:28 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3566502934' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T10:44:30.741 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:44:30.741 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":59,"fsid":"ddf74116-74e0-11ef-bceb-c7b262605968","created":"2024-09-17T10:38:16.982968+0000","modified":"2024-09-17T10:44:09.808633+0000","last_up_change":"2024-09-17T10:44:06.753584+0000","last_in_change":"2024-09-17T10:43:49.875759+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T10:42:12.008115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"2fbbec5b-85fb-4bd7-82ef-adc149c87960","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6803","nonce":3722513843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6805","nonce":3722513843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6809","nonce":3722513843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":3722513843},{"type":"v1","addr":"172.21.15.6:6807","nonce":3722513843}]},"public_addr":"172.21.15.6:6803/3722513843","cluster_addr":"172.21.15.6:6805/3722513843","heartbeat_back_addr":"172.21.15.6:6809/3722513843","heartbeat_front_addr":"172.21.15.6:6807/3722513843","state":["exists","up"]},{"osd":1,"uuid":"927193ac-f68c-463b-b254-a0096ab97bbb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6811","nonce":3766560025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6813","nonce":3766560025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6817","nonce":3766560025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":3766560025},{"type":"v1","addr":"172.21.15.6:6815","nonce":3766560025}]},"public_addr":"172.21.15.6:6811/3766560025","cluster_addr":"172.21.15.6:6813/3766560025","heartbeat_back_addr":"172.21.15.6:6817/3766560025","heartbeat_front_addr":"172.21.15.6:6815/3766560025","state":["exists","up"]},{"osd":2,"uuid":"ef897e3f-bf8c-4430-a3a6-ead5208c6a51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6819","nonce":969866728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6821","nonce":969866728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6825","nonce":969866728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":969866728},{"type":"v1","addr":"172.21.15.6:6823","nonce":969866728}]},"public_addr":"172.21.15.6:6819/969866728","cluster_addr":"172.21.15.6:6821/969866728","heartbeat_back_addr":"172.21.15.6:6825/969866728","heartbeat_front_addr":"172.21.15.6:6823/969866728","state":["exists","up"]},{"osd":3,"uuid":"b004b6a3-05fd-487c-9dd0-3f685bd51cdc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6827","nonce":2766264617}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6829","nonce":2766264617}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6833","nonce":2766264617}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":2766264617},{"type":"v1","addr":"172.21.15.6:6831","nonce":2766264617}]},"public_addr":"172.21.15.6:6827/2766264617","cluster_addr":"172.21.15.6:6829/2766264617","heartbeat_back_addr":"172.21.15.6:6833/2766264617","heartbeat_front_addr":"172.21.15.6:6831/2766264617","state":["exists","up"]},{"osd":4,"uuid":"aecefec1-82ec-4259-aed3-8d4d06ee5ce2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6801","nonce":761527016}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6803","nonce":761527016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6807","nonce":761527016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":761527016},{"type":"v1","addr":"172.21.15.149:6805","nonce":761527016}]},"public_addr":"172.21.15.149:6801/761527016","cluster_addr":"172.21.15.149:6803/761527016","heartbeat_back_addr":"172.21.15.149:6807/761527016","heartbeat_front_addr":"172.21.15.149:6805/761527016","state":["exists","up"]},{"osd":5,"uuid":"12c7592b-af31-44a4-b872-26fb12b6932b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6809","nonce":1397443451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6811","nonce":1397443451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6815","nonce":1397443451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":1397443451},{"type":"v1","addr":"172.21.15.149:6813","nonce":1397443451}]},"public_addr":"172.21.15.149:6809/1397443451","cluster_addr":"172.21.15.149:6811/1397443451","heartbeat_back_addr":"172.21.15.149:6815/1397443451","heartbeat_front_addr":"172.21.15.149:6813/1397443451","state":["exists","up"]},{"osd":6,"uuid":"3863943d-9cdf-47b9-bf2a-0641a445e2a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6817","nonce":3219489945}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6819","nonce":3219489945}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6823","nonce":3219489945}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":3219489945},{"type":"v1","addr":"172.21.15.149:6821","nonce":3219489945}]},"public_addr":"172.21.15.149:6817/3219489945","cluster_addr":"172.21.15.149:6819/3219489945","heartbeat_back_addr":"172.21.15.149:6823/3219489945","heartbeat_front_addr":"172.21.15.149:6821/3219489945","state":["exists","up"]},{"osd":7,"uuid":"979b6e41-161a-45a2-bc8b-1dd980e78bf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6825","nonce":2180530477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6827","nonce":2180530477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6831","nonce":2180530477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":2180530477},{"type":"v1","addr":"172.21.15.149:6829","nonce":2180530477}]},"public_addr":"172.21.15.149:6825/2180530477","cluster_addr":"172.21.15.149:6827/2180530477","heartbeat_back_addr":"172.21.15.149:6831/2180530477","heartbeat_front_addr":"172.21.15.149:6829/2180530477","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:41:11.883494+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:41:42.408147+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:08.990744+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:34.229094+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:42:55.844856+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:43:18.190182+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:43:41.095017+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T10:44:04.979017+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/271712424":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/3791438495":"2024-09-18T10:40:01.448980+0000","172.21.15.6:6801/411654524":"2024-09-18T10:40:01.448980+0000","172.21.15.6:6800/411654524":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/194568551":"2024-09-18T10:40:01.448980+0000","172.21.15.6:0/1873228168":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6800/2106963250":"2024-09-18T10:39:12.608826+0000","172.21.15.6:0/1062781985":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/2550476919":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/1670227312":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6801/2106963250":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6800/3033656982":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/2729884814":"2024-09-18T10:38:42.872777+0000","172.21.15.6:0/3074268711":"2024-09-18T10:39:12.608826+0000","172.21.15.6:6801/3033656982":"2024-09-18T10:38:42.872777+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T10:44:31.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:30 smithi006 ceph-mon[23740]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:31.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:30 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1198284680' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T10:44:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:30 smithi149 ceph-mon[26044]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:30 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1198284680' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T10:44:31.468 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-17T10:44:31.469 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-17T10:44:31.469 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-17T10:44:31.470 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-17T10:44:31.470 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-17T10:44:31.470 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-17T10:44:31.471 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-17T10:44:31.472 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-17T10:44:31.895 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:31.899 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:31.899 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:31.900 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:31.901 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:31.901 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:31.908 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:31.933 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:32.935 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:32 smithi006 ceph-mon[23740]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:32 smithi149 ceph-mon[26044]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:35.066 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:34 smithi006 ceph-mon[23740]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:34 smithi149 ceph-mon[26044]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:37.032 INFO:teuthology.orchestra.run.smithi006.stdout:201863462925 2024-09-17T10:44:37.033 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-17T10:44:37.131 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:36 smithi006 ceph-mon[23740]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:37.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:36 smithi149 ceph-mon[26044]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:38.633 INFO:teuthology.orchestra.run.smithi006.stdout:133143986198 2024-09-17T10:44:38.633 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-17T10:44:38.858 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:38 smithi006 ceph-mon[23740]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:38 smithi149 ceph-mon[26044]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:40.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:40 smithi006 ceph-mon[23740]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:41.172 INFO:teuthology.orchestra.run.smithi006.stdout:167503724562 2024-09-17T10:44:41.173 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-17T10:44:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:40 smithi149 ceph-mon[26044]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:42.304 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:42.519 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:43.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:42 smithi006 ceph-mon[23740]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:42 smithi149 ceph-mon[26044]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:45.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:44 smithi006 ceph-mon[23740]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:45.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:44 smithi149 ceph-mon[26044]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:45.460 INFO:teuthology.orchestra.run.smithi006.stdout:236223201290 2024-09-17T10:44:45.460 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-17T10:44:47.012 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:46 smithi006 ceph-mon[23740]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:47.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:46 smithi149 ceph-mon[26044]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:48.027 INFO:teuthology.orchestra.run.smithi006.stdout:60129542184 2024-09-17T10:44:48.028 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-17T10:44:48.680 INFO:teuthology.orchestra.run.smithi006.stdout:111669149725 2024-09-17T10:44:48.681 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-17T10:44:48.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:48 smithi006 ceph-mon[23740]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:49.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:48 smithi149 ceph-mon[26044]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:49.306 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:50.213 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:50.213 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:50.720 INFO:teuthology.orchestra.run.smithi006.stdout:81604378659 2024-09-17T10:44:50.720 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-17T10:44:50.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:50 smithi006 ceph-mon[23740]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:50 smithi149 ceph-mon[26044]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:52.005 INFO:teuthology.orchestra.run.smithi006.stdout:38654705710 2024-09-17T10:44:52.005 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-17T10:44:52.946 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:53.128 INFO:teuthology.orchestra.run.smithi006.stdout:201863462928 2024-09-17T10:44:53.129 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:52 smithi006 ceph-mon[23740]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:52 smithi149 ceph-mon[26044]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:53.256 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:54.104 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:53 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/196096485' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T10:44:54.181 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:53 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/196096485' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T10:44:54.910 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:54 smithi006 ceph-mon[23740]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:54 smithi149 ceph-mon[26044]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:56.062 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:44:56.467 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462925 got 201863462928 for osd.6 2024-09-17T10:44:56.467 DEBUG:teuthology.parallel:result is None 2024-09-17T10:44:56.868 INFO:teuthology.orchestra.run.smithi006.stdout:133143986202 2024-09-17T10:44:57.187 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:56 smithi006 ceph-mon[23740]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:57.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:56 smithi149 ceph-mon[26044]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:58.189 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:57 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3998891587' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T10:44:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:57 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3998891587' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T10:44:58.268 INFO:teuthology.orchestra.run.smithi006.stdout:167503724566 2024-09-17T10:44:58.671 INFO:teuthology.orchestra.run.smithi006.stdout:236223201293 2024-09-17T10:44:58.940 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986198 got 133143986202 for osd.4 2024-09-17T10:44:58.940 DEBUG:teuthology.parallel:result is None 2024-09-17T10:44:59.130 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:58 smithi006 ceph-mon[23740]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:59.130 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:58 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1367570688' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T10:44:59.130 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:44:58 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1255933288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T10:44:59.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:58 smithi149 ceph-mon[26044]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:44:59.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:58 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1367570688' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T10:44:59.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:44:58 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1255933288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T10:44:59.982 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724562 got 167503724566 for osd.5 2024-09-17T10:44:59.983 DEBUG:teuthology.parallel:result is None 2024-09-17T10:45:01.005 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:00 smithi006 ceph-mon[23740]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:01.034 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201290 got 236223201293 for osd.7 2024-09-17T10:45:01.034 DEBUG:teuthology.parallel:result is None 2024-09-17T10:45:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:00 smithi149 ceph-mon[26044]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:02.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:45:02.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:01 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:45:02.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:45:02.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:01 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:45:02.939 INFO:teuthology.orchestra.run.smithi006.stdout:60129542187 2024-09-17T10:45:03.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:02 smithi006 ceph-mon[23740]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:02 smithi149 ceph-mon[26044]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:03.538 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542187 for osd.1 2024-09-17T10:45:03.538 DEBUG:teuthology.parallel:result is None 2024-09-17T10:45:03.933 INFO:teuthology.orchestra.run.smithi006.stdout:81604378662 2024-09-17T10:45:03.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:03 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/316865222' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T10:45:04.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:03 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/316865222' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T10:45:04.671 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378659 got 81604378662 for osd.2 2024-09-17T10:45:04.671 DEBUG:teuthology.parallel:result is None 2024-09-17T10:45:04.921 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:04 smithi006 ceph-mon[23740]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:04.921 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:04 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2194747692' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T10:45:05.116 INFO:teuthology.orchestra.run.smithi006.stdout:111669149729 2024-09-17T10:45:05.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:04 smithi149 ceph-mon[26044]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:05.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:04 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/2194747692' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T10:45:05.951 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149729 for osd.3 2024-09-17T10:45:05.951 DEBUG:teuthology.parallel:result is None 2024-09-17T10:45:06.033 INFO:teuthology.orchestra.run.smithi006.stdout:38654705714 2024-09-17T10:45:06.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:05 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3790009757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T10:45:06.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:05 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3790009757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T10:45:06.668 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705710 got 38654705714 for osd.0 2024-09-17T10:45:06.669 DEBUG:teuthology.parallel:result is None 2024-09-17T10:45:06.669 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-17T10:45:06.669 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T10:45:06.891 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:45:06.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:06 smithi006 ceph-mon[23740]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:06.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:06 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1687275868' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T10:45:07.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:06 smithi149 ceph-mon[26044]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:07.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:06 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1687275868' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T10:45:08.681 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:45:08.682 INFO:teuthology.orchestra.run.smithi006.stderr:dumped all 2024-09-17T10:45:09.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:08 smithi006 ceph-mon[23740]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:09.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:09.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:09.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:09.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:08 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:08 smithi149 ceph-mon[26044]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:08 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:09.373 INFO:teuthology.orchestra.run.smithi006.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2024-09-17T10:45:07.554692+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48796,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912420,"statfs":{"total":767960285184,"available":767910318080,"internally_reserved":0,"allocated":6451200,"data_stored":3546719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002048"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-17T10:44:09.814584+0000","last_change":"2024-09-17T10:44:09.814584+0000","last_active":"2024-09-17T10:44:09.814584+0000","last_peered":"2024-09-17T10:44:09.814584+0000","last_clean":"2024-09-17T10:44:09.814584+0000","last_became_active":"2024-09-17T10:44:09.814108+0000","last_became_peered":"2024-09-17T10:44:09.814108+0000","last_unstale":"2024-09-17T10:44:09.814584+0000","last_undegraded":"2024-09-17T10:44:09.814584+0000","last_fullsized":"2024-09-17T10:44:09.814584+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T10:42:12.008334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T10:42:12.008334+0000","last_clean_scrub_stamp":"2024-09-17T10:42:12.008334+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T21:50:11.892285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54200000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.501}]},{"osd":1,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.62}]},{"osd":2,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.55200000000000005}]},{"osd":3,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.67600000000000005}]},{"osd":4,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.48599999999999999}]},{"osd":5,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.438},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.52800000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70799999999999996}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6356,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738796,"statfs":{"total":95995035648,"available":95988527104,"internally_reserved":0,"allocated":602112,"data_stored":244703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:52 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.34200000000000003,"15min":0.32200000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.33400000000000002},"min":{"1min":0.25600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.49199999999999999}]},{"osd":2,"last update":"Tue Sep 17 10:44:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.38100000000000001,"15min":0.377},"min":{"1min":0.26800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.51500000000000001,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.438,"5min":0.40999999999999998,"15min":0.40600000000000003},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.52900000000000003}]},{"osd":3,"last update":"Tue Sep 17 10:44:43 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.47299999999999998,"15min":0.48599999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.54400000000000004,"5min":0.622,"15min":0.622},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.443,"5min":0.48999999999999999,"15min":0.498},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.58199999999999996}]},{"osd":4,"last update":"Tue Sep 17 10:45:02 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58299999999999996,"15min":0.58399999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.71999999999999997,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.621,"5min":0.60299999999999998,"15min":0.59999999999999998},"min":{"1min":0.47699999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.75800000000000001}]},{"osd":5,"last update":"Tue Sep 17 10:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.78900000000000003}]},{"osd":6,"last update":"Tue Sep 17 10:44:43 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.73199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82399999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 10:44:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.38900000000000001,"15min":0.38300000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":0.76600000000000001,"5min":1.383,"15min":1.383},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.34599999999999997},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64800000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.50800000000000001}]},{"osd":2,"last update":"Tue Sep 17 10:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.36899999999999999,"15min":0.35899999999999999},"min":{"1min":0.318,"5min":0.25,"15min":0.25},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.373},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.38,"15min":0.36499999999999999},"min":{"1min":0.25,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.58699999999999997}]},{"osd":3,"last update":"Tue Sep 17 10:44:40 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.49199999999999999,"15min":0.49399999999999999},"min":{"1min":0.32300000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.438,"5min":0.45400000000000001,"15min":0.45700000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.43099999999999999}]},{"osd":4,"last update":"Tue Sep 17 10:44:59 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.61899999999999999,"15min":0.61399999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.95799999999999996,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.65100000000000002,"15min":0.64900000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.95699999999999996,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.65700000000000003}]},{"osd":5,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"last":0.86299999999999999}]},{"osd":6,"last update":"Tue Sep 17 10:44:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.68300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.89700000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6232,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:16 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.39200000000000002,"15min":0.38200000000000001},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.42599999999999999,"15min":0.42199999999999999},"min":{"1min":0.32400000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.58999999999999997,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.377}]},{"osd":1,"last update":"Tue Sep 17 10:44:16 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.42199999999999999,"15min":0.41599999999999998},"min":{"1min":0.313,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57499999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.42899999999999999,"15min":0.42399999999999999},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.38800000000000001}]},{"osd":3,"last update":"Tue Sep 17 10:44:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.498,"15min":0.502},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.67300000000000004,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.41299999999999998}]},{"osd":4,"last update":"Tue Sep 17 10:44:59 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.41999999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.61499999999999999,"15min":0.61699999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.76400000000000001}]},{"osd":5,"last update":"Tue Sep 17 10:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.67700000000000005}]},{"osd":6,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.74099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69499999999999995}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:42 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.45200000000000001,"15min":0.44800000000000001},"min":{"1min":0.32600000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.46500000000000002,"15min":0.46700000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.35599999999999998}]},{"osd":1,"last update":"Tue Sep 17 10:44:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.47299999999999998,"15min":0.47499999999999998},"min":{"1min":0.318,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.59499999999999997,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.47899999999999998,"15min":0.48099999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.46600000000000003}]},{"osd":2,"last update":"Tue Sep 17 10:44:42 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51400000000000001,"15min":0.51300000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47999999999999998,"15min":0.48199999999999998},"min":{"1min":0.34799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.432}]},{"osd":4,"last update":"Tue Sep 17 10:45:01 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.80500000000000005,"15min":0.83199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0529999999999999,"5min":5.1890000000000001,"15min":5.1890000000000001},"last":0.503},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.64600000000000002,"15min":0.64300000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.51700000000000002}]},{"osd":5,"last update":"Tue Sep 17 10:44:23 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.495},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.60499999999999998}]},{"osd":6,"last update":"Tue Sep 17 10:44:48 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.083,"5min":1.083,"15min":1.083},"last":0.621}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:43:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.52100000000000002}]},{"osd":1,"last update":"Tue Sep 17 10:43:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.48399999999999999}]},{"osd":2,"last update":"Tue Sep 17 10:43:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.57199999999999995}]},{"osd":3,"last update":"Tue Sep 17 10:43:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.50900000000000001}]},{"osd":5,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"last":0.42299999999999999}]},{"osd":6,"last update":"Tue Sep 17 10:44:48 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.33700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58699999999999997}]},{"osd":1,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.51300000000000001}]},{"osd":2,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.61299999999999999}]},{"osd":3,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.63},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.52000000000000002}]},{"osd":4,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.44}]},{"osd":6,"last update":"Tue Sep 17 10:44:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.497},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.48599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T10:45:09.374 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T10:45:09.596 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:45:10.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:09 smithi006 ceph-mon[23740]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:10.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:09 smithi149 ceph-mon[26044]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:11.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:10 smithi006 ceph-mon[23740]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:10 smithi149 ceph-mon[26044]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:11.251 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:45:11.253 INFO:teuthology.orchestra.run.smithi006.stderr:dumped all 2024-09-17T10:45:11.877 INFO:teuthology.orchestra.run.smithi006.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-17T10:45:09.555177+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48796,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912420,"statfs":{"total":767960285184,"available":767910318080,"internally_reserved":0,"allocated":6451200,"data_stored":3546719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002192"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-17T10:44:09.814584+0000","last_change":"2024-09-17T10:44:09.814584+0000","last_active":"2024-09-17T10:44:09.814584+0000","last_peered":"2024-09-17T10:44:09.814584+0000","last_clean":"2024-09-17T10:44:09.814584+0000","last_became_active":"2024-09-17T10:44:09.814108+0000","last_became_peered":"2024-09-17T10:44:09.814108+0000","last_unstale":"2024-09-17T10:44:09.814584+0000","last_undegraded":"2024-09-17T10:44:09.814584+0000","last_fullsized":"2024-09-17T10:44:09.814584+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T10:42:12.008334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T10:42:12.008334+0000","last_clean_scrub_stamp":"2024-09-17T10:42:12.008334+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T21:50:11.892285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54200000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462932,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.435}]},{"osd":1,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.53600000000000003}]},{"osd":2,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.52400000000000002}]},{"osd":3,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.5}]},{"osd":4,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.375}]},{"osd":5,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.41899999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6356,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738796,"statfs":{"total":95995035648,"available":95988527104,"internally_reserved":0,"allocated":602112,"data_stored":244703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:52 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.34200000000000003,"15min":0.32200000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.33400000000000002},"min":{"1min":0.25600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.49199999999999999}]},{"osd":2,"last update":"Tue Sep 17 10:44:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.38100000000000001,"15min":0.377},"min":{"1min":0.26800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.51500000000000001,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.438,"5min":0.40999999999999998,"15min":0.40600000000000003},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.52900000000000003}]},{"osd":3,"last update":"Tue Sep 17 10:44:43 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.47299999999999998,"15min":0.48599999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.54400000000000004,"5min":0.622,"15min":0.622},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.443,"5min":0.48999999999999999,"15min":0.498},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.58199999999999996}]},{"osd":4,"last update":"Tue Sep 17 10:45:02 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58299999999999996,"15min":0.58399999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.71999999999999997,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.621,"5min":0.60299999999999998,"15min":0.59999999999999998},"min":{"1min":0.47699999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.75800000000000001}]},{"osd":5,"last update":"Tue Sep 17 10:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.78900000000000003}]},{"osd":6,"last update":"Tue Sep 17 10:44:43 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.73199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82399999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 10:44:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.38900000000000001,"15min":0.38300000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":0.76600000000000001,"5min":1.383,"15min":1.383},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.34599999999999997},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64800000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.50800000000000001}]},{"osd":2,"last update":"Tue Sep 17 10:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.36899999999999999,"15min":0.35899999999999999},"min":{"1min":0.318,"5min":0.25,"15min":0.25},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.373},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.38,"15min":0.36499999999999999},"min":{"1min":0.25,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.58699999999999997}]},{"osd":3,"last update":"Tue Sep 17 10:44:40 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.49199999999999999,"15min":0.49399999999999999},"min":{"1min":0.32300000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.438,"5min":0.45400000000000001,"15min":0.45700000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.43099999999999999}]},{"osd":4,"last update":"Tue Sep 17 10:44:59 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.61899999999999999,"15min":0.61399999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.95799999999999996,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.65100000000000002,"15min":0.64900000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.95699999999999996,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.65700000000000003}]},{"osd":5,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"last":0.86299999999999999}]},{"osd":6,"last update":"Tue Sep 17 10:44:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.68300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.89700000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6232,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:16 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.39200000000000002,"15min":0.38200000000000001},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.42599999999999999,"15min":0.42199999999999999},"min":{"1min":0.32400000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.58999999999999997,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.377}]},{"osd":1,"last update":"Tue Sep 17 10:44:16 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.42199999999999999,"15min":0.41599999999999998},"min":{"1min":0.313,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57499999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.42899999999999999,"15min":0.42399999999999999},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.38800000000000001}]},{"osd":3,"last update":"Tue Sep 17 10:44:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.498,"15min":0.502},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.67300000000000004,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.41299999999999998}]},{"osd":4,"last update":"Tue Sep 17 10:44:59 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.41999999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.61499999999999999,"15min":0.61699999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.76400000000000001}]},{"osd":5,"last update":"Tue Sep 17 10:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.67700000000000005}]},{"osd":6,"last update":"Tue Sep 17 10:44:46 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.74099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69499999999999995}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:42 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.45200000000000001,"15min":0.44800000000000001},"min":{"1min":0.32600000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.46500000000000002,"15min":0.46700000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.35599999999999998}]},{"osd":1,"last update":"Tue Sep 17 10:44:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.47299999999999998,"15min":0.47499999999999998},"min":{"1min":0.318,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.59499999999999997,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.47899999999999998,"15min":0.48099999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.46600000000000003}]},{"osd":2,"last update":"Tue Sep 17 10:44:42 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51400000000000001,"15min":0.51300000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47999999999999998,"15min":0.48199999999999998},"min":{"1min":0.34799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.432}]},{"osd":4,"last update":"Tue Sep 17 10:45:01 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.80500000000000005,"15min":0.83199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0529999999999999,"5min":5.1890000000000001,"15min":5.1890000000000001},"last":0.503},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.64600000000000002,"15min":0.64300000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.51700000000000002}]},{"osd":5,"last update":"Tue Sep 17 10:44:23 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.495},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.60499999999999998}]},{"osd":6,"last update":"Tue Sep 17 10:44:48 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.083,"5min":1.083,"15min":1.083},"last":0.621}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:45:03 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52900000000000003,"15min":0.53000000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40899999999999997,"5min":0.372,"15min":0.372},"max":{"1min":0.755,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.53800000000000003}]},{"osd":1,"last update":"Tue Sep 17 10:45:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.36499999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.78100000000000003,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.80700000000000005,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.50600000000000001}]},{"osd":2,"last update":"Tue Sep 17 10:45:03 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.371,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.75700000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.76900000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.66300000000000003}]},{"osd":3,"last update":"Tue Sep 17 10:45:03 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.56200000000000006,"15min":0.55900000000000005},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.82499999999999996,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.41999999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.67600000000000005}]},{"osd":5,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"last":0.64600000000000002}]},{"osd":6,"last update":"Tue Sep 17 10:44:48 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.63200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69299999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58699999999999997}]},{"osd":1,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.51300000000000001}]},{"osd":2,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.61299999999999999}]},{"osd":3,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.63},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.52000000000000002}]},{"osd":4,"last update":"Tue Sep 17 10:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.44}]},{"osd":6,"last update":"Tue Sep 17 10:44:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.497},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.48599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T10:45:11.878 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-17T10:45:11.879 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-17T10:45:11.879 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-17T10:45:11.879 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-17T10:45:12.097 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/mon.smithi006/config 2024-09-17T10:45:13.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:12 smithi006 ceph-mon[23740]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:13.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:12 smithi006 ceph-mon[23740]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:12 smithi149 ceph-mon[26044]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:12 smithi149 ceph-mon[26044]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:13.986 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:45:13.986 INFO:teuthology.orchestra.run.smithi006.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-17T10:45:14.533 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-17T10:45:14.533 INFO:tasks.cephadm:Setup complete, yielding 2024-09-17T10:45:14.534 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T10:45:14.545 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:45:14.545 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-17T10:45:15.144 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:14 smithi006 ceph-mon[23740]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:15.144 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:14 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/402846618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T10:45:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:14 smithi149 ceph-mon[26044]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:15.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:14 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/402846618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T10:45:16.432 INFO:teuthology.orchestra.run.smithi006.stdout:Backend: cephadm 2024-09-17T10:45:16.433 INFO:teuthology.orchestra.run.smithi006.stdout:Available: Yes 2024-09-17T10:45:16.433 INFO:teuthology.orchestra.run.smithi006.stdout:Paused: No 2024-09-17T10:45:17.055 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-17T10:45:17.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:16 smithi006 ceph-mon[23740]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:16 smithi149 ceph-mon[26044]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:18.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:17 smithi006 ceph-mon[23740]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:17 smithi149 ceph-mon[26044]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:18.997 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:45:18.997 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c b74431e6f177 2024-09-17T10:45:18.997 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:45:18.997 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (5m) 73s ago 5m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (4m) 2m ago 5m 43.6M - 8.3.5 dad864ee21e9 fcaa29cab62f 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:9283 running (6m) 2m ago 6m 447M - 17.2.0 e1d6a67b021e 8caa6368de72 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283 running (5m) 73s ago 5m 411M - 17.2.0 e1d6a67b021e 34dc4cd55cb3 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (6m) 2m ago 7m 44.3M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (4m) 73s ago 4m 38.9M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (5m) 2m ago 5m 15.4M - 1dbe0e931976 19b622692f8e 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (4m) 73s ago 4m 17.2M - 1dbe0e931976 b3c75ec0fcd4 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (4m) 2m ago 4m 32.3M 3217M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (3m) 2m ago 3m 34.1M 3217M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (3m) 2m ago 3m 30.4M 3217M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (2m) 2m ago 2m 10.9M 3217M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:45:18.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (2m) 73s ago 2m 32.8M 1165M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:45:18.999 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (2m) 73s ago 2m 33.1M 1165M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:45:18.999 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (102s) 73s ago 102s 33.3M 1165M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:45:18.999 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (78s) 73s ago 78s 9.99M 1165M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:45:18.999 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (4m) 2m ago 4m 55.9M - 514e6a882f6e f5cf8e877b8a 2024-09-17T10:45:19.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:18 smithi006 ceph-mon[23740]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:19.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:18 smithi149 ceph-mon[26044]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:19.637 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-17T10:45:20.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:19 smithi006 ceph-mon[23740]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:20.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:19 smithi149 ceph-mon[26044]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:21.121 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:20 smithi006 ceph-mon[23740]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:21.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:20 smithi149 ceph-mon[26044]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:21.544 INFO:teuthology.orchestra.run.smithi006.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-17T10:45:21.544 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-17T10:45:21.545 INFO:teuthology.orchestra.run.smithi006.stdout:crash 2/2 2m ago 6m * 2024-09-17T10:45:21.545 INFO:teuthology.orchestra.run.smithi006.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-17T10:45:21.545 INFO:teuthology.orchestra.run.smithi006.stdout:mgr 2/2 2m ago 6m count:2 2024-09-17T10:45:21.545 INFO:teuthology.orchestra.run.smithi006.stdout:mon 2/2 2m ago 5m smithi006:172.21.15.6=smithi006;smithi149:172.21.15.149=smithi149;count:2 2024-09-17T10:45:21.545 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-17T10:45:21.545 INFO:teuthology.orchestra.run.smithi006.stdout:osd 8 2m ago - 2024-09-17T10:45:21.545 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-17T10:45:22.158 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-17T10:45:23.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:22 smithi006 ceph-mon[23740]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:23.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:22 smithi006 ceph-mon[23740]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:23.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:22 smithi149 ceph-mon[26044]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:23.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:22 smithi149 ceph-mon[26044]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:24.080 INFO:teuthology.orchestra.run.smithi006.stdout:HOST ADDR LABELS STATUS 2024-09-17T10:45:24.080 INFO:teuthology.orchestra.run.smithi006.stdout:smithi006 172.21.15.6 2024-09-17T10:45:24.080 INFO:teuthology.orchestra.run.smithi006.stdout:smithi149 172.21.15.149 2024-09-17T10:45:24.080 INFO:teuthology.orchestra.run.smithi006.stdout:2 hosts in cluster 2024-09-17T10:45:24.681 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-17T10:45:25.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:24 smithi006 ceph-mon[23740]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:25.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:24 smithi006 ceph-mon[23740]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:25.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:24 smithi149 ceph-mon[26044]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:25.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:24 smithi149 ceph-mon[26044]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:26.678 INFO:teuthology.orchestra.run.smithi006.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-17T10:45:26.679 INFO:teuthology.orchestra.run.smithi006.stdout:smithi006 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5330000H400BGN 400G LVM detected, locked 2024-09-17T10:45:26.679 INFO:teuthology.orchestra.run.smithi006.stdout:smithi149 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300CK400BGN 400G LVM detected, locked 2024-09-17T10:45:27.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:26 smithi006 ceph-mon[23740]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:27.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:26 smithi149 ceph-mon[26044]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:27.323 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T10:45:27.334 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:45:27.335 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-17T10:45:27.365 INFO:teuthology.orchestra.run.smithi006.stderr:+ systemctl stop nfs-server 2024-09-17T10:45:27.373 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi149.front.sepia.ceph.com 2024-09-17T10:45:27.373 DEBUG:teuthology.orchestra.run.smithi149:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-17T10:45:27.403 INFO:teuthology.orchestra.run.smithi149.stderr:+ systemctl stop nfs-server 2024-09-17T10:45:27.411 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T10:45:27.420 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:45:27.420 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-17T10:45:28.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:27 smithi006 ceph-mon[23740]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:28.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:27 smithi149 ceph-mon[26044]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:29.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:28 smithi006 ceph-mon[23740]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:28 smithi149 ceph-mon[26044]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:30.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:29 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-17T10:45:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:29 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-17T10:45:31.368 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:30 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[23736]: 2024-09-17T10:45:30.959+0000 7f3dd423e700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:45:31.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:30 smithi006 ceph-mon[23740]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:31.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:30 smithi006 ceph-mon[23740]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:31.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-17T10:45:31.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:30 smithi006 ceph-mon[23740]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T10:45:31.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:30 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-17T10:45:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:30 smithi149 ceph-mon[26044]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:30 smithi149 ceph-mon[26044]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-17T10:45:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:30 smithi149 ceph-mon[26044]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T10:45:31.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:30 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-17T10:45:31.775 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-17T10:45:31.784 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-17T10:45:31.784 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: fsmap foofs:0 2024-09-17T10:45:32.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: Saving service mds.foofs spec with placement count:2 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.nripkg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.nripkg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: Deploying daemon mds.foofs.smithi006.nripkg on smithi006 2024-09-17T10:45:32.102 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:31 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-17T10:45:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T10:45:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: fsmap foofs:0 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: Saving service mds.foofs spec with placement count:2 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:32.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.nripkg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T10:45:32.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.nripkg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T10:45:32.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:32.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: Deploying daemon mds.foofs.smithi006.nripkg on smithi006 2024-09-17T10:45:32.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:31 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:33.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:32 smithi006 ceph-mon[23740]: pgmap v215: 65 pgs: 11 creating+peering, 1 active+clean, 53 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:33.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:32 smithi006 ceph-mon[23740]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T10:45:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:32 smithi149 ceph-mon[26044]: pgmap v215: 65 pgs: 11 creating+peering, 1 active+clean, 53 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:32 smithi149 ceph-mon[26044]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T10:45:34.348 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:33 smithi006 ceph-mon[23740]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T10:45:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:33 smithi149 ceph-mon[26044]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T10:45:34.983 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:45:34.983 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T10:39:01.555849Z", "last_refresh": "2024-09-17T10:42:37.041306Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T10:38:55.442162Z", "last_refresh": "2024-09-17T10:42:37.041440Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T10:38:58.348685Z", "last_refresh": "2024-09-17T10:42:37.041558Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T10:45:30.985389Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T10:45:30.981172Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T10:38:54.230645Z", "last_refresh": "2024-09-17T10:42:37.041198Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T10:39:46.116436Z", "last_refresh": "2024-09-17T10:42:37.041027Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T10:38:59.564645Z", "last_refresh": "2024-09-17T10:42:37.041628Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T10:42:37.041758Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T10:38:56.617323Z", "last_refresh": "2024-09-17T10:42:37.041694Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T10:45:35.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:34 smithi006 ceph-mon[23740]: pgmap v218: 65 pgs: 38 active+clean, 11 creating+peering, 16 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:35.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:34 smithi149 ceph-mon[26044]: pgmap v218: 65 pgs: 38 active+clean, 11 creating+peering, 16 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:35.599 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-17T10:45:36.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:35 smithi006 ceph-mon[23740]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:36.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:35 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:36.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:35 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.hmuwiq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T10:45:36.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:35 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.hmuwiq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T10:45:36.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:35 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:36.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:35 smithi149 ceph-mon[26044]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:36.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:35 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:36.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:35 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.hmuwiq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T10:45:36.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:35 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.hmuwiq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T10:45:36.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:35 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:36.600 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T10:45:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: Deploying daemon mds.foofs.smithi149.hmuwiq on smithi149 2024-09-17T10:45:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: mds.? [v2:172.21.15.6:6834/1356880066,v1:172.21.15.6:6835/1356880066] up:boot 2024-09-17T10:45:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: daemon mds.foofs.smithi006.nripkg assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-17T10:45:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T10:45:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-17T10:45:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: Cluster is now healthy 2024-09-17T10:45:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: fsmap foofs:0 1 up:standby 2024-09-17T10:45:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"}]: dispatch 2024-09-17T10:45:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:creating} 2024-09-17T10:45:37.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:36 smithi006 ceph-mon[23740]: daemon mds.foofs.smithi006.nripkg is now active in filesystem foofs as rank 0 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: Deploying daemon mds.foofs.smithi149.hmuwiq on smithi149 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: mds.? [v2:172.21.15.6:6834/1356880066,v1:172.21.15.6:6835/1356880066] up:boot 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: daemon mds.foofs.smithi006.nripkg assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: Cluster is now healthy 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: fsmap foofs:0 1 up:standby 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"}]: dispatch 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:creating} 2024-09-17T10:45:37.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:36 smithi149 ceph-mon[26044]: daemon mds.foofs.smithi006.nripkg is now active in filesystem foofs as rank 0 2024-09-17T10:45:38.033 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:37 smithi149 ceph-mon[26044]: mds.? [v2:172.21.15.6:6834/1356880066,v1:172.21.15.6:6835/1356880066] up:active 2024-09-17T10:45:38.034 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:37 smithi149 ceph-mon[26044]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 2024-09-17T10:45:38.144 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:37 smithi006 ceph-mon[23740]: mds.? [v2:172.21.15.6:6834/1356880066,v1:172.21.15.6:6835/1356880066] up:active 2024-09-17T10:45:38.144 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:37 smithi006 ceph-mon[23740]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 2024-09-17T10:45:38.663 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:45:38.663 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T10:39:01.555849Z", "last_refresh": "2024-09-17T10:42:37.041306Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T10:38:55.442162Z", "last_refresh": "2024-09-17T10:42:37.041440Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T10:38:58.348685Z", "last_refresh": "2024-09-17T10:42:37.041558Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T10:45:30.985389Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T10:45:30.981172Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T10:38:54.230645Z", "last_refresh": "2024-09-17T10:42:37.041198Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T10:39:46.116436Z", "last_refresh": "2024-09-17T10:42:37.041027Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T10:38:59.564645Z", "last_refresh": "2024-09-17T10:42:37.041628Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T10:42:37.041758Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T10:38:56.617323Z", "last_refresh": "2024-09-17T10:42:37.041694Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T10:45:39.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: pgmap v220: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 3 op/s 2024-09-17T10:45:39.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: mds.? [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] up:boot 2024-09-17T10:45:39.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:45:39.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"}]: dispatch 2024-09-17T10:45:39.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:45:39.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:39.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:39.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:39.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:38 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:39.290 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: pgmap v220: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 3 op/s 2024-09-17T10:45:39.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: mds.? [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] up:boot 2024-09-17T10:45:39.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:45:39.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"}]: dispatch 2024-09-17T10:45:39.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:45:39.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:39.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:39.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:39.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:38 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:39.563 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-17T10:45:40.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:40 smithi006 ceph-mon[23740]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:40.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:40 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:40 smithi149 ceph-mon[26044]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:40 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:40.565 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T10:45:41.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:41 smithi006 ceph-mon[23740]: pgmap v221: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 3 op/s 2024-09-17T10:45:41.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:41 smithi149 ceph-mon[26044]: pgmap v221: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 3 op/s 2024-09-17T10:45:42.859 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:42 smithi006 ceph-mon[23740]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-17T10:45:42.859 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:42 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:42.920 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:42 smithi149 ceph-mon[26044]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-17T10:45:42.920 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:42 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:43.309 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:45:43.309 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T10:39:01.555849Z", "last_refresh": "2024-09-17T10:42:37.041306Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T10:38:55.442162Z", "last_refresh": "2024-09-17T10:42:37.041440Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T10:38:58.348685Z", "last_refresh": "2024-09-17T10:42:37.041558Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T10:45:30.985389Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T10:45:30.981172Z", "last_refresh": "2024-09-17T10:45:39.951033Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T10:38:54.230645Z", "last_refresh": "2024-09-17T10:42:37.041198Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T10:39:46.116436Z", "last_refresh": "2024-09-17T10:42:37.041027Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T10:38:59.564645Z", "last_refresh": "2024-09-17T10:42:37.041628Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T10:42:37.041758Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T10:38:56.617323Z", "last_refresh": "2024-09-17T10:42:37.041694Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T10:45:44.506 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-17T10:45:44.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:44 smithi006 ceph-mon[23740]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:44.874 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:44 smithi006 ceph-mon[23740]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-17T10:45:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:44 smithi149 ceph-mon[26044]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:44 smithi149 ceph-mon[26044]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-17T10:45:45.507 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T10:45:46.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:46.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:46.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:46.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:46.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:46.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:46 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:46.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:46 smithi006 ceph-mon[23740]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T10:45:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:46 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:46 smithi149 ceph-mon[26044]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T10:45:47.492 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:45:47.492 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T10:39:01.555849Z", "last_refresh": "2024-09-17T10:45:45.391279Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T10:38:55.442162Z", "last_refresh": "2024-09-17T10:45:39.950359Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T10:38:58.348685Z", "last_refresh": "2024-09-17T10:45:45.391406Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T10:45:30.985389Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T10:45:30.981172Z", "last_refresh": "2024-09-17T10:45:39.951033Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T10:38:54.230645Z", "last_refresh": "2024-09-17T10:45:39.950515Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T10:39:46.116436Z", "last_refresh": "2024-09-17T10:45:39.950591Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T10:38:59.564645Z", "last_refresh": "2024-09-17T10:45:39.950683Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T10:45:39.950749Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T10:38:56.617323Z", "last_refresh": "2024-09-17T10:45:45.391530Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T10:45:47.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:47 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:47 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:48.210 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-17T10:45:48.211 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T10:45:48.223 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:45:48.223 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-17T10:45:48.861 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:48 smithi006 ceph-mon[23740]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:48.861 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:48 smithi006 ceph-mon[23740]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T10:45:48.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:48 smithi149 ceph-mon[26044]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T10:45:48.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:48 smithi149 ceph-mon[26044]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T10:45:50.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:50 smithi149 ceph-mon[26044]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-09-17T10:45:50.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:50 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-17T10:45:50.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:50 smithi006 ceph-mon[23740]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-09-17T10:45:50.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:50 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-17T10:45:51.780 INFO:teuthology.orchestra.run.smithi006.stdout:NFS Cluster Created Successfully 2024-09-17T10:45:52.105 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:51 smithi006 ceph-mon[23740]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:52.105 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:51 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-17T10:45:52.105 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:51 smithi006 ceph-mon[23740]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T10:45:52.106 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:51 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-17T10:45:52.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:51 smithi149 ceph-mon[26044]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:52.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:51 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-17T10:45:52.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:51 smithi149 ceph-mon[26044]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T10:45:52.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:51 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-17T10:45:52.508 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: pgmap v228: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: Saving service nfs.foo spec with placement count:2 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:53.109 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: Creating key for client.nfs.foo.0.0.smithi006.izithr 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: Rados config object exists: conf-nfs.foo 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: Creating key for client.nfs.foo.0.0.smithi006.izithr-rgw 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T10:45:53.110 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:53.111 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:52 smithi149 ceph-mon[26044]: Deploying daemon nfs.foo.0.0.smithi006.izithr on smithi006 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: pgmap v228: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: Saving service nfs.foo spec with placement count:2 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:45:53.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: Creating key for client.nfs.foo.0.0.smithi006.izithr 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: Rados config object exists: conf-nfs.foo 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: Creating key for client.nfs.foo.0.0.smithi006.izithr-rgw 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T10:45:53.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T10:45:53.204 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:53.204 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:52 smithi006 ceph-mon[23740]: Deploying daemon nfs.foo.0.0.smithi006.izithr on smithi006 2024-09-17T10:45:53.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:53 smithi149 ceph-mon[26044]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T10:45:54.067 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:53 smithi006 ceph-mon[23740]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T10:45:55.129 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:54 smithi006 ceph-mon[23740]: pgmap v231: 97 pgs: 12 unknown, 85 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T10:45:55.129 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:54 smithi006 ceph-mon[23740]: mgrmap e20: smithi006.plhvdm(active, since 5m), standbys: smithi149.ijckag 2024-09-17T10:45:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:54 smithi149 ceph-mon[26044]: pgmap v231: 97 pgs: 12 unknown, 85 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T10:45:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:54 smithi149 ceph-mon[26044]: mgrmap e20: smithi006.plhvdm(active, since 5m), standbys: smithi149.ijckag 2024-09-17T10:45:55.792 INFO:teuthology.orchestra.run.smithi006.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-17T10:45:55.792 INFO:teuthology.orchestra.run.smithi006.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-17T10:45:55.793 INFO:teuthology.orchestra.run.smithi006.stderr:Error EINVAL: invalid command 2024-09-17T10:45:56.183 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:45:56.183 INFO:teuthology.orchestra.run.smithi006.stdout: "bind": "/fake", 2024-09-17T10:45:56.183 INFO:teuthology.orchestra.run.smithi006.stdout: "fs": "foofs", 2024-09-17T10:45:56.183 INFO:teuthology.orchestra.run.smithi006.stdout: "path": "/", 2024-09-17T10:45:56.183 INFO:teuthology.orchestra.run.smithi006.stdout: "cluster": "foo", 2024-09-17T10:45:56.184 INFO:teuthology.orchestra.run.smithi006.stdout: "mode": "RW" 2024-09-17T10:45:56.184 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:45:56.807 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-17T10:45:57.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: Creating key for client.nfs.foo.1.0.smithi149.ppflwc 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-17T10:45:57.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:57 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-17T10:45:57.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T10:45:57.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:45:57.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: Creating key for client.nfs.foo.1.0.smithi149.ppflwc 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-17T10:45:57.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:57 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-17T10:45:59.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:59 smithi006 ceph-mon[23740]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-17T10:45:59.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:59 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T10:45:59.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:59 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T10:45:59.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:59 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T10:45:59.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:59 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T10:45:59.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:45:59 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:45:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:59 smithi149 ceph-mon[26044]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-17T10:45:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:59 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T10:45:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:59 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T10:45:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:59 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T10:45:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:59 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T10:45:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:45:59 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:46:00.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:00 smithi006 ceph-mon[23740]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:00.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:00 smithi006 ceph-mon[23740]: Rados config object exists: conf-nfs.foo 2024-09-17T10:46:00.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:00 smithi006 ceph-mon[23740]: Creating key for client.nfs.foo.1.0.smithi149.ppflwc-rgw 2024-09-17T10:46:00.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:00 smithi006 ceph-mon[23740]: Deploying daemon nfs.foo.1.0.smithi149.ppflwc on smithi149 2024-09-17T10:46:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:00 smithi149 ceph-mon[26044]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:00 smithi149 ceph-mon[26044]: Rados config object exists: conf-nfs.foo 2024-09-17T10:46:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:00 smithi149 ceph-mon[26044]: Creating key for client.nfs.foo.1.0.smithi149.ppflwc-rgw 2024-09-17T10:46:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:00 smithi149 ceph-mon[26044]: Deploying daemon nfs.foo.1.0.smithi149.ppflwc on smithi149 2024-09-17T10:46:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:01 smithi149 ceph-mon[26044]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-17T10:46:01.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:01 smithi006 ceph-mon[23740]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-17T10:46:02.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:02.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]: dispatch 2024-09-17T10:46:02.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-17T10:46:02.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-17T10:46:02.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-17T10:46:02.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:46:02.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:46:02.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:02.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:46:02.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:46:02.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:46:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]: dispatch 2024-09-17T10:46:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-17T10:46:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-17T10:46:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-17T10:46:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:46:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:46:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:46:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:46:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:46:03.189 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:03 smithi149 ceph-mon[26044]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-17T10:46:03.190 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:03 smithi149 ceph-mon[26044]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:03.190 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:03 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]': finished 2024-09-17T10:46:03.190 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:03 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-17T10:46:03.190 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:03 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-17T10:46:03.190 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:03 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-17T10:46:03.190 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:03 smithi149 ceph-mon[26044]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T10:46:03.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:03 smithi006 ceph-mon[23740]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-17T10:46:03.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:03 smithi006 ceph-mon[23740]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:03.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:03 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]': finished 2024-09-17T10:46:03.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:03 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-17T10:46:03.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:03 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-17T10:46:03.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:03 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-17T10:46:03.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:03 smithi006 ceph-mon[23740]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T10:46:04.355 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:04 smithi149 ceph-mon[26044]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:04.355 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:04 smithi149 ceph-mon[26044]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T10:46:04.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:04 smithi006 ceph-mon[23740]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:04.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:04 smithi006 ceph-mon[23740]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T10:46:05.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:05 smithi006 ceph-mon[23740]: pgmap v238: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-17T10:46:05.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:05 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:05 smithi149 ceph-mon[26044]: pgmap v238: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-17T10:46:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:05 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:06.183 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:06 smithi006 ceph-mon[23740]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:06.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:06 smithi149 ceph-mon[26044]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:06.908 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:07 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-17T10:46:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:46:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:46:07.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:46:07.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:07 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:07.598 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T10:46:07.607 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:46:07.608 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-17T10:46:07.636 INFO:teuthology.orchestra.run.smithi006.stderr:+ mkdir /mnt/foo 2024-09-17T10:46:07.649 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-17T10:46:07.716 INFO:teuthology.orchestra.run.smithi006.stderr:++ hostname 2024-09-17T10:46:07.716 INFO:teuthology.orchestra.run.smithi006.stderr:+ mount -t nfs smithi006:/fake /mnt/foo -o sync 2024-09-17T10:46:08.159 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-17T10:46:08.225 INFO:teuthology.orchestra.run.smithi006.stderr:+ echo test 2024-09-17T10:46:08.235 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-17T10:46:08.297 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:08 smithi006 ceph-mon[23740]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:08.300 INFO:teuthology.orchestra.run.smithi006.stderr:+ sync 2024-09-17T10:46:08.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:08 smithi149 ceph-mon[26044]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:08.533 INFO:teuthology.run_tasks:Running task parallel... 2024-09-17T10:46:08.543 INFO:teuthology.task.parallel:starting parallel... 2024-09-17T10:46:08.543 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-17T10:46:08.544 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-17T10:46:08.544 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:46:08.544 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-17T10:46:08.544 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-17T10:46:08.545 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-17T10:46:08.546 INFO:teuthology.task.exec:Executing custom commands... 2024-09-17T10:46:08.546 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:46:08.546 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-17T10:46:08.672 INFO:teuthology.orchestra.run.smithi006.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T10:46:08.672 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:46:08.672 INFO:teuthology.orchestra.run.smithi006.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-17T10:46:08.673 INFO:teuthology.orchestra.run.smithi006.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-17T10:46:08.773 INFO:teuthology.orchestra.run.smithi006.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T10:46:08.773 INFO:teuthology.orchestra.run.smithi006.stdout:releasing clients 2024-09-17T10:46:09.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:09 smithi006 ceph-mon[23740]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-17T10:46:09.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:09 smithi149 ceph-mon[26044]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-17T10:46:09.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 86 19.92 MB/sec warmup 1 sec latency 49.079 ms 2024-09-17T10:46:10.471 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:10 smithi006 ceph-mon[23740]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-17T10:46:10.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:10 smithi149 ceph-mon[26044]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-17T10:46:10.773 INFO:teuthology.orchestra.run.smithi006.stdout: 5 206 20.16 MB/sec warmup 2 sec latency 39.224 ms 2024-09-17T10:46:11.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 442 25.84 MB/sec warmup 3 sec latency 38.782 ms 2024-09-17T10:46:12.285 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-17T10:46:12.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 672 26.46 MB/sec warmup 4 sec latency 38.745 ms 2024-09-17T10:46:13.026 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:12 smithi006 ceph-mon[23740]: pgmap v242: 97 pgs: 97 active+clean; 6.1 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 603 KiB/s wr, 15 op/s 2024-09-17T10:46:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:12 smithi149 ceph-mon[26044]: pgmap v242: 97 pgs: 97 active+clean; 6.1 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 603 KiB/s wr, 15 op/s 2024-09-17T10:46:13.773 INFO:teuthology.orchestra.run.smithi006.stdout: 5 1026 22.73 MB/sec warmup 5 sec latency 53.626 ms 2024-09-17T10:46:14.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 1684 21.85 MB/sec warmup 6 sec latency 23.862 ms 2024-09-17T10:46:14.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:14 smithi006 ceph-mon[23740]: pgmap v243: 97 pgs: 97 active+clean; 60 MiB data, 109 MiB used, 715 GiB / 715 GiB avail; 196 B/s rd, 5.9 MiB/s wr, 280 op/s 2024-09-17T10:46:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:14 smithi149 ceph-mon[26044]: pgmap v243: 97 pgs: 97 active+clean; 60 MiB data, 109 MiB used, 715 GiB / 715 GiB avail; 196 B/s rd, 5.9 MiB/s wr, 280 op/s 2024-09-17T10:46:15.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 2851 23.19 MB/sec warmup 7 sec latency 10.819 ms 2024-09-17T10:46:16.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 3762 24.66 MB/sec warmup 8 sec latency 13.953 ms 2024-09-17T10:46:17.062 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:16 smithi006 ceph-mon[23740]: pgmap v244: 97 pgs: 97 active+clean; 96 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 8.4 MiB/s wr, 323 op/s 2024-09-17T10:46:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:16 smithi149 ceph-mon[26044]: pgmap v244: 97 pgs: 97 active+clean; 96 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 8.4 MiB/s wr, 323 op/s 2024-09-17T10:46:17.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 4076 22.28 MB/sec warmup 9 sec latency 44.021 ms 2024-09-17T10:46:17.882 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-17T10:46:18.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 4593 20.74 MB/sec warmup 10 sec latency 55.507 ms 2024-09-17T10:46:18.862 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:18 smithi006 ceph-mon[23740]: pgmap v245: 97 pgs: 97 active+clean; 120 MiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 11 MiB/s wr, 592 op/s 2024-09-17T10:46:19.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:18 smithi149 ceph-mon[26044]: pgmap v245: 97 pgs: 97 active+clean; 120 MiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 11 MiB/s wr, 592 op/s 2024-09-17T10:46:19.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 5461 20.70 MB/sec warmup 11 sec latency 44.340 ms 2024-09-17T10:46:20.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 5942 19.62 MB/sec warmup 12 sec latency 40.536 ms 2024-09-17T10:46:20.875 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:20 smithi006 ceph-mon[23740]: pgmap v246: 97 pgs: 97 active+clean; 124 MiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 12 MiB/s wr, 630 op/s 2024-09-17T10:46:21.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:20 smithi149 ceph-mon[26044]: pgmap v246: 97 pgs: 97 active+clean; 124 MiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 12 MiB/s wr, 630 op/s 2024-09-17T10:46:21.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 6577 20.27 MB/sec warmup 13 sec latency 61.180 ms 2024-09-17T10:46:22.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 7230 20.72 MB/sec warmup 14 sec latency 43.380 ms 2024-09-17T10:46:22.903 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:22 smithi006 ceph-mon[23740]: pgmap v247: 97 pgs: 97 active+clean; 133 MiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 14 MiB/s wr, 767 op/s 2024-09-17T10:46:23.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:22 smithi149 ceph-mon[26044]: pgmap v247: 97 pgs: 97 active+clean; 133 MiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 14 MiB/s wr, 767 op/s 2024-09-17T10:46:23.710 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-17T10:46:23.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 7473 19.71 MB/sec warmup 15 sec latency 54.118 ms 2024-09-17T10:46:24.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 7880 18.63 MB/sec warmup 16 sec latency 49.737 ms 2024-09-17T10:46:24.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:24 smithi006 ceph-mon[23740]: pgmap v248: 97 pgs: 97 active+clean; 137 MiB data, 297 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 15 MiB/s wr, 991 op/s 2024-09-17T10:46:25.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:24 smithi149 ceph-mon[26044]: pgmap v248: 97 pgs: 97 active+clean; 137 MiB data, 297 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 15 MiB/s wr, 991 op/s 2024-09-17T10:46:25.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 8228 18.00 MB/sec warmup 17 sec latency 42.126 ms 2024-09-17T10:46:26.774 INFO:teuthology.orchestra.run.smithi006.stdout: 5 8651 17.87 MB/sec warmup 18 sec latency 58.673 ms 2024-09-17T10:46:26.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:26 smithi006 ceph-mon[23740]: pgmap v249: 97 pgs: 97 active+clean; 139 MiB data, 325 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 12 MiB/s wr, 798 op/s 2024-09-17T10:46:27.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:26 smithi149 ceph-mon[26044]: pgmap v249: 97 pgs: 97 active+clean; 139 MiB data, 325 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 12 MiB/s wr, 798 op/s 2024-09-17T10:46:27.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 9401 17.33 MB/sec warmup 19 sec latency 20.601 ms 2024-09-17T10:46:27.877 INFO:teuthology.orchestra.run.smithi006.stderr:module 'nfs' is already enabled 2024-09-17T10:46:27.979 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:27 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1942794536' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-17T10:46:28.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:27 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1942794536' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-17T10:46:28.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 10028 17.88 MB/sec warmup 20 sec latency 43.060 ms 2024-09-17T10:46:28.992 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:28 smithi006 ceph-mon[23740]: pgmap v250: 97 pgs: 97 active+clean; 145 MiB data, 345 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 930 op/s 2024-09-17T10:46:28.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:28 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1942794536' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-17T10:46:28.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:28 smithi006 ceph-mon[23740]: mgrmap e21: smithi006.plhvdm(active, since 6m), standbys: smithi149.ijckag 2024-09-17T10:46:29.124 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T10:46:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:28 smithi149 ceph-mon[26044]: pgmap v250: 97 pgs: 97 active+clean; 145 MiB data, 345 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 930 op/s 2024-09-17T10:46:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:28 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1942794536' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-17T10:46:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:28 smithi149 ceph-mon[26044]: mgrmap e21: smithi006.plhvdm(active, since 6m), standbys: smithi149.ijckag 2024-09-17T10:46:29.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 10508 17.82 MB/sec warmup 21 sec latency 53.786 ms 2024-09-17T10:46:30.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 10870 17.72 MB/sec warmup 22 sec latency 75.616 ms 2024-09-17T10:46:31.028 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:30 smithi006 ceph-mon[23740]: pgmap v251: 97 pgs: 97 active+clean; 143 MiB data, 355 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 678 op/s 2024-09-17T10:46:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:30 smithi149 ceph-mon[26044]: pgmap v251: 97 pgs: 97 active+clean; 143 MiB data, 355 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 678 op/s 2024-09-17T10:46:31.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 11171 17.03 MB/sec warmup 23 sec latency 62.896 ms 2024-09-17T10:46:32.617 INFO:teuthology.orchestra.run.smithi006.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T10:46:32.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 11505 16.44 MB/sec warmup 24 sec latency 77.301 ms 2024-09-17T10:46:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:32 smithi149 ceph-mon[26044]: pgmap v252: 97 pgs: 97 active+clean; 146 MiB data, 395 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 727 op/s 2024-09-17T10:46:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:46:32.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:46:32.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:32 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:46:33.051 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:32 smithi006 ceph-mon[23740]: pgmap v252: 97 pgs: 97 active+clean; 146 MiB data, 395 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 727 op/s 2024-09-17T10:46:33.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:33.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:46:33.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:46:33.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:32 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:46:33.736 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-17T10:46:33.737 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T10:46:33.737 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-17T10:46:33.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 11795 16.07 MB/sec warmup 25 sec latency 36.020 ms 2024-09-17T10:46:34.086 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:34 smithi006 ceph-mon[23740]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:34.087 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:34 smithi006 ceph-mon[23740]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T10:46:34.087 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:34 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:34.087 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:34 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:34.087 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:34 smithi006 ceph-mon[23740]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T10:46:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:34 smithi149 ceph-mon[26044]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:34 smithi149 ceph-mon[26044]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T10:46:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:34 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:34 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:46:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:34 smithi149 ceph-mon[26044]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T10:46:34.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 12344 16.07 MB/sec warmup 26 sec latency 77.280 ms 2024-09-17T10:46:35.091 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:35 smithi006 ceph-mon[23740]: pgmap v253: 97 pgs: 97 active+clean; 152 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 718 op/s 2024-09-17T10:46:35.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:35 smithi149 ceph-mon[26044]: pgmap v253: 97 pgs: 97 active+clean; 152 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 718 op/s 2024-09-17T10:46:35.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 12887 15.62 MB/sec warmup 27 sec latency 66.800 ms 2024-09-17T10:46:36.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 13209 15.49 MB/sec warmup 28 sec latency 32.463 ms 2024-09-17T10:46:37.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:37 smithi006 ceph-mon[23740]: pgmap v254: 97 pgs: 97 active+clean; 152 MiB data, 455 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.8 MiB/s wr, 643 op/s 2024-09-17T10:46:37.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:37 smithi149 ceph-mon[26044]: pgmap v254: 97 pgs: 97 active+clean; 152 MiB data, 455 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.8 MiB/s wr, 643 op/s 2024-09-17T10:46:37.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 13750 15.75 MB/sec warmup 29 sec latency 24.103 ms 2024-09-17T10:46:38.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 14290 16.00 MB/sec warmup 30 sec latency 36.138 ms 2024-09-17T10:46:39.120 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:39 smithi006 ceph-mon[23740]: pgmap v255: 97 pgs: 97 active+clean; 156 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 740 op/s 2024-09-17T10:46:39.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:39 smithi149 ceph-mon[26044]: pgmap v255: 97 pgs: 97 active+clean; 156 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 740 op/s 2024-09-17T10:46:39.775 INFO:teuthology.orchestra.run.smithi006.stdout: 5 14588 15.71 MB/sec warmup 31 sec latency 38.183 ms 2024-09-17T10:46:40.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 15264 15.47 MB/sec warmup 32 sec latency 33.637 ms 2024-09-17T10:46:41.128 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:41 smithi006 ceph-mon[23740]: pgmap v256: 97 pgs: 97 active+clean; 155 MiB data, 475 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.7 MiB/s wr, 547 op/s 2024-09-17T10:46:41.190 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:46:41.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:41 smithi149 ceph-mon[26044]: pgmap v256: 97 pgs: 97 active+clean; 155 MiB data, 475 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.7 MiB/s wr, 547 op/s 2024-09-17T10:46:41.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 15792 15.55 MB/sec warmup 33 sec latency 40.263 ms 2024-09-17T10:46:42.143 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:42 smithi006 ceph-mon[23740]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (5m) 35s ago 7m 22.3M - ba2b418f427c b74431e6f177 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (7m) 35s ago 7m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (6m) 37s ago 6m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (5m) 35s ago 6m 55.3M - 8.3.5 dad864ee21e9 fcaa29cab62f 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (67s) 35s ago 66s 14.7M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (64s) 37s ago 64s 12.0M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:9283 running (8m) 35s ago 8m 459M - 17.2.0 e1d6a67b021e 8caa6368de72 2024-09-17T10:46:42.299 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283 running (6m) 37s ago 6m 412M - 17.2.0 e1d6a67b021e 34dc4cd55cb3 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (8m) 35s ago 8m 49.5M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (6m) 37s ago 6m 36.5M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (46s) 35s ago 46s 58.2M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (41s) 37s ago 40s 50.7M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (6m) 35s ago 6m 15.8M - 1dbe0e931976 19b622692f8e 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (6m) 37s ago 6m 16.8M - 1dbe0e931976 b3c75ec0fcd4 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (5m) 35s ago 5m 43.7M 3217M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (5m) 35s ago 5m 43.0M 3217M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (4m) 35s ago 4m 39.4M 3217M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (4m) 35s ago 4m 41.4M 3217M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (3m) 37s ago 3m 39.8M 1165M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:46:42.300 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (3m) 37s ago 3m 40.1M 1165M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:46:42.301 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (3m) 37s ago 3m 41.7M 1165M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:46:42.301 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (2m) 37s ago 2m 39.3M 1165M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:46:42.301 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (6m) 35s ago 6m 62.9M - 514e6a882f6e f5cf8e877b8a 2024-09-17T10:46:42.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:42 smithi149 ceph-mon[26044]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:42.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 16485 15.27 MB/sec warmup 34 sec latency 47.231 ms 2024-09-17T10:46:42.982 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:46:42.982 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:46:42.982 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:46:42.982 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:46:42.983 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:46:42.984 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T10:46:42.984 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:46:42.984 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:46:43.149 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:43 smithi006 ceph-mon[23740]: pgmap v257: 97 pgs: 97 active+clean; 166 MiB data, 516 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.9 MiB/s wr, 680 op/s 2024-09-17T10:46:43.149 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:43 smithi006 ceph-mon[23740]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:43.150 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:43 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1247216541' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:46:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:43 smithi149 ceph-mon[26044]: pgmap v257: 97 pgs: 97 active+clean; 166 MiB data, 516 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.9 MiB/s wr, 680 op/s 2024-09-17T10:46:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:43 smithi149 ceph-mon[26044]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:43 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1247216541' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:46:43.549 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:46:43.549 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169", 2024-09-17T10:46:43.549 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:46:43.549 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-09-17T10:46:43.549 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "", 2024-09-17T10:46:43.549 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 image" 2024-09-17T10:46:43.549 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:46:43.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 17061 15.59 MB/sec warmup 35 sec latency 26.768 ms 2024-09-17T10:46:44.159 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:44 smithi006 ceph-mon[23740]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:44.240 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:46:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:44 smithi149 ceph-mon[26044]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:44.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 17791 15.98 MB/sec warmup 36 sec latency 23.729 ms 2024-09-17T10:46:45.166 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:45 smithi006 ceph-mon[23740]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:45.167 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:45 smithi006 ceph-mon[23740]: pgmap v258: 97 pgs: 97 active+clean; 170 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 682 op/s 2024-09-17T10:46:45.167 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:45 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2955098400' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:46:45.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:45 smithi149 ceph-mon[26044]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:46:45.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:45 smithi149 ceph-mon[26044]: pgmap v258: 97 pgs: 97 active+clean; 170 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 682 op/s 2024-09-17T10:46:45.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:45 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/2955098400' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:46:45.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 18253 15.75 MB/sec warmup 37 sec latency 24.155 ms 2024-09-17T10:46:46.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 18939 15.68 MB/sec warmup 38 sec latency 16.102 ms 2024-09-17T10:46:47.179 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:47 smithi006 ceph-mon[23740]: pgmap v259: 97 pgs: 97 active+clean; 177 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.3 MiB/s wr, 744 op/s 2024-09-17T10:46:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:47 smithi149 ceph-mon[26044]: pgmap v259: 97 pgs: 97 active+clean; 177 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.3 MiB/s wr, 744 op/s 2024-09-17T10:46:47.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 19894 15.70 MB/sec warmup 39 sec latency 21.546 ms 2024-09-17T10:46:48.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 20710 16.05 MB/sec warmup 40 sec latency 18.982 ms 2024-09-17T10:46:49.333 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:49 smithi006 ceph-mon[23740]: pgmap v260: 97 pgs: 97 active+clean; 180 MiB data, 613 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 697 op/s 2024-09-17T10:46:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:49 smithi149 ceph-mon[26044]: pgmap v260: 97 pgs: 97 active+clean; 180 MiB data, 613 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 697 op/s 2024-09-17T10:46:49.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 21472 16.43 MB/sec warmup 41 sec latency 19.393 ms 2024-09-17T10:46:50.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 21950 16.18 MB/sec warmup 42 sec latency 24.909 ms 2024-09-17T10:46:51.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:51 smithi006 ceph-mon[23740]: pgmap v261: 97 pgs: 97 active+clean; 180 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.0 MiB/s wr, 584 op/s 2024-09-17T10:46:51.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:51 smithi149 ceph-mon[26044]: pgmap v261: 97 pgs: 97 active+clean; 180 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.0 MiB/s wr, 584 op/s 2024-09-17T10:46:51.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 22698 16.17 MB/sec warmup 43 sec latency 14.526 ms 2024-09-17T10:46:52.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 23523 16.13 MB/sec warmup 44 sec latency 70.791 ms 2024-09-17T10:46:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:53 smithi149 ceph-mon[26044]: pgmap v262: 97 pgs: 97 active+clean; 186 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 8.7 MiB/s wr, 806 op/s 2024-09-17T10:46:53.353 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:53 smithi006 ceph-mon[23740]: pgmap v262: 97 pgs: 97 active+clean; 186 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 8.7 MiB/s wr, 806 op/s 2024-09-17T10:46:53.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 24186 16.26 MB/sec warmup 45 sec latency 29.560 ms 2024-09-17T10:46:54.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 25070 16.73 MB/sec warmup 46 sec latency 15.528 ms 2024-09-17T10:46:55.360 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:55 smithi006 ceph-mon[23740]: pgmap v263: 97 pgs: 97 active+clean; 197 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 908 op/s 2024-09-17T10:46:55.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:55 smithi149 ceph-mon[26044]: pgmap v263: 97 pgs: 97 active+clean; 197 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 908 op/s 2024-09-17T10:46:55.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 25611 16.51 MB/sec warmup 47 sec latency 13.543 ms 2024-09-17T10:46:56.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 26259 16.48 MB/sec warmup 48 sec latency 16.971 ms 2024-09-17T10:46:57.375 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:57 smithi006 ceph-mon[23740]: pgmap v264: 97 pgs: 97 active+clean; 196 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 882 op/s 2024-09-17T10:46:57.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:57 smithi149 ceph-mon[26044]: pgmap v264: 97 pgs: 97 active+clean; 196 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 882 op/s 2024-09-17T10:46:57.776 INFO:teuthology.orchestra.run.smithi006.stdout: 5 26964 16.42 MB/sec warmup 49 sec latency 33.618 ms 2024-09-17T10:46:58.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 27611 16.47 MB/sec warmup 50 sec latency 26.666 ms 2024-09-17T10:46:59.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:46:59 smithi006 ceph-mon[23740]: pgmap v265: 97 pgs: 97 active+clean; 200 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.7 MiB/s wr, 845 op/s 2024-09-17T10:46:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:46:59 smithi149 ceph-mon[26044]: pgmap v265: 97 pgs: 97 active+clean; 200 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.7 MiB/s wr, 845 op/s 2024-09-17T10:46:59.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 28396 16.77 MB/sec warmup 51 sec latency 26.595 ms 2024-09-17T10:47:00.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 28975 16.74 MB/sec warmup 52 sec latency 21.045 ms 2024-09-17T10:47:01.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:01 smithi006 ceph-mon[23740]: pgmap v266: 97 pgs: 97 active+clean; 205 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 852 op/s 2024-09-17T10:47:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:01 smithi149 ceph-mon[26044]: pgmap v266: 97 pgs: 97 active+clean; 205 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 852 op/s 2024-09-17T10:47:01.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 29591 16.67 MB/sec warmup 53 sec latency 16.383 ms 2024-09-17T10:47:02.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:47:02.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:47:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:47:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:47:02.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 30263 16.61 MB/sec warmup 54 sec latency 17.355 ms 2024-09-17T10:47:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:03 smithi149 ceph-mon[26044]: pgmap v267: 97 pgs: 97 active+clean; 212 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 985 op/s 2024-09-17T10:47:03.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:03 smithi006 ceph-mon[23740]: pgmap v267: 97 pgs: 97 active+clean; 212 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 985 op/s 2024-09-17T10:47:03.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 30864 16.48 MB/sec warmup 55 sec latency 35.703 ms 2024-09-17T10:47:04.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 31517 16.70 MB/sec warmup 56 sec latency 28.259 ms 2024-09-17T10:47:05.370 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:05 smithi006 ceph-mon[23740]: pgmap v268: 97 pgs: 97 active+clean; 220 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-09-17T10:47:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:05 smithi149 ceph-mon[26044]: pgmap v268: 97 pgs: 97 active+clean; 220 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-09-17T10:47:05.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 32166 16.87 MB/sec warmup 57 sec latency 16.709 ms 2024-09-17T10:47:06.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 32743 16.69 MB/sec warmup 58 sec latency 43.065 ms 2024-09-17T10:47:07.387 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:07 smithi006 ceph-mon[23740]: pgmap v269: 97 pgs: 97 active+clean; 221 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 856 op/s 2024-09-17T10:47:07.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:07 smithi149 ceph-mon[26044]: pgmap v269: 97 pgs: 97 active+clean; 221 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 856 op/s 2024-09-17T10:47:07.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 33282 16.64 MB/sec warmup 59 sec latency 48.005 ms 2024-09-17T10:47:08.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 33687 16.54 MB/sec warmup 60 sec latency 56.575 ms 2024-09-17T10:47:09.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:09 smithi006 ceph-mon[23740]: pgmap v270: 97 pgs: 97 active+clean; 224 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 937 op/s 2024-09-17T10:47:09.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:09 smithi149 ceph-mon[26044]: pgmap v270: 97 pgs: 97 active+clean; 224 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 937 op/s 2024-09-17T10:47:09.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 34127 16.33 MB/sec warmup 61 sec latency 52.988 ms 2024-09-17T10:47:10.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 34583 16.31 MB/sec warmup 62 sec latency 52.861 ms 2024-09-17T10:47:11.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:11 smithi006 ceph-mon[23740]: pgmap v271: 97 pgs: 97 active+clean; 224 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 822 op/s 2024-09-17T10:47:11.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:11 smithi149 ceph-mon[26044]: pgmap v271: 97 pgs: 97 active+clean; 224 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 822 op/s 2024-09-17T10:47:11.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 35289 16.52 MB/sec warmup 63 sec latency 64.253 ms 2024-09-17T10:47:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 35718 16.52 MB/sec warmup 64 sec latency 91.522 ms 2024-09-17T10:47:13.263 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:13 smithi006 ceph-mon[23740]: pgmap v272: 97 pgs: 97 active+clean; 221 MiB data, 992 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 823 op/s 2024-09-17T10:47:13.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:13 smithi149 ceph-mon[26044]: pgmap v272: 97 pgs: 97 active+clean; 221 MiB data, 992 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 823 op/s 2024-09-17T10:47:13.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 35851 16.31 MB/sec warmup 65 sec latency 82.934 ms 2024-09-17T10:47:14.642 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:47:14.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 36101 16.09 MB/sec warmup 66 sec latency 97.140 ms 2024-09-17T10:47:14.817 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:14 smithi006 ceph-mon[23740]: pgmap v273: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.5 MiB/s wr, 927 op/s 2024-09-17T10:47:14.817 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:47:14.817 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:14 smithi006 ceph-mon[23740]: Upgrade: Target is version 19.3.0-4937-g74984932 (unknown) 2024-09-17T10:47:14.817 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:14 smithi006 ceph-mon[23740]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb'] 2024-09-17T10:47:14.818 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:47:14.818 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:14 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:47:14.818 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:14 smithi006 ceph-mon[23740]: Upgrade: Need to upgrade myself (mgr.smithi006.plhvdm) 2024-09-17T10:47:14.818 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:14 smithi006 ceph-mon[23740]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb on smithi149 2024-09-17T10:47:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:14 smithi149 ceph-mon[26044]: pgmap v273: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.5 MiB/s wr, 927 op/s 2024-09-17T10:47:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:47:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:14 smithi149 ceph-mon[26044]: Upgrade: Target is version 19.3.0-4937-g74984932 (unknown) 2024-09-17T10:47:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:14 smithi149 ceph-mon[26044]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb'] 2024-09-17T10:47:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:47:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:14 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:47:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:14 smithi149 ceph-mon[26044]: Upgrade: Need to upgrade myself (mgr.smithi006.plhvdm) 2024-09-17T10:47:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:14 smithi149 ceph-mon[26044]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb on smithi149 2024-09-17T10:47:15.484 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:47:15.484 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (6m) 69s ago 7m 22.3M - ba2b418f427c b74431e6f177 2024-09-17T10:47:15.484 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (7m) 69s ago 7m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:47:15.484 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (7m) 71s ago 7m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (6m) 69s ago 7m 55.3M - 8.3.5 dad864ee21e9 fcaa29cab62f 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (100s) 69s ago 99s 14.7M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (97s) 71s ago 97s 12.0M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:9283 running (8m) 69s ago 8m 459M - 17.2.0 e1d6a67b021e 8caa6368de72 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283 running (7m) 71s ago 7m 412M - 17.2.0 e1d6a67b021e 34dc4cd55cb3 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (8m) 69s ago 8m 49.5M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (6m) 71s ago 6m 36.5M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (79s) 69s ago 79s 58.2M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:47:15.485 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (74s) 71s ago 73s 50.7M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (7m) 69s ago 7m 15.8M - 1dbe0e931976 19b622692f8e 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (6m) 71s ago 6m 16.8M - 1dbe0e931976 b3c75ec0fcd4 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (6m) 69s ago 6m 43.7M 3217M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (5m) 69s ago 5m 43.0M 3217M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (5m) 69s ago 5m 39.4M 3217M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (4m) 69s ago 4m 41.4M 3217M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (4m) 71s ago 4m 39.8M 1165M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (4m) 71s ago 4m 40.1M 1165M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (3m) 71s ago 3m 41.7M 1165M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (3m) 71s ago 3m 39.3M 1165M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:47:15.486 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (6m) 69s ago 6m 62.9M - 514e6a882f6e f5cf8e877b8a 2024-09-17T10:47:15.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 36497 15.94 MB/sec warmup 67 sec latency 93.477 ms 2024-09-17T10:47:15.827 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:15 smithi006 ceph-mon[23740]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:15.828 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:15 smithi006 ceph-mon[23740]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:16.158 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:47:16.158 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:47:16.158 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:47:16.158 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:47:16.158 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:47:16.158 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:47:16.158 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:47:16.158 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:47:16.159 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:47:16.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:15 smithi149 ceph-mon[26044]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:16.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:15 smithi149 ceph-mon[26044]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:16.719 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:47:16.719 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:47:16.719 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:47:16.719 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-09-17T10:47:16.719 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "0/23 daemons upgraded", 2024-09-17T10:47:16.719 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb image on host smithi149" 2024-09-17T10:47:16.719 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:47:16.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 37102 15.94 MB/sec warmup 68 sec latency 33.391 ms 2024-09-17T10:47:16.831 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:16 smithi006 ceph-mon[23740]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:16.831 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:16 smithi006 ceph-mon[23740]: pgmap v274: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 650 op/s 2024-09-17T10:47:16.831 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:16 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2530995994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:47:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:16 smithi149 ceph-mon[26044]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:16 smithi149 ceph-mon[26044]: pgmap v274: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 650 op/s 2024-09-17T10:47:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:16 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/2530995994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:47:17.374 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:47:17.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 37779 15.85 MB/sec warmup 69 sec latency 27.209 ms 2024-09-17T10:47:18.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:17 smithi006 ceph-mon[23740]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:18.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:17 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/356623356' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:47:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:17 smithi149 ceph-mon[26044]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:17 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/356623356' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:47:18.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 38231 15.86 MB/sec warmup 70 sec latency 71.518 ms 2024-09-17T10:47:18.851 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:18 smithi006 ceph-mon[23740]: pgmap v275: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.1 MiB/s wr, 699 op/s 2024-09-17T10:47:19.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:18 smithi149 ceph-mon[26044]: pgmap v275: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.1 MiB/s wr, 699 op/s 2024-09-17T10:47:19.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 38644 15.89 MB/sec warmup 71 sec latency 49.948 ms 2024-09-17T10:47:20.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 39114 15.94 MB/sec warmup 72 sec latency 61.521 ms 2024-09-17T10:47:20.881 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:20 smithi006 ceph-mon[23740]: pgmap v276: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 621 op/s 2024-09-17T10:47:21.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:20 smithi149 ceph-mon[26044]: pgmap v276: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 621 op/s 2024-09-17T10:47:21.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 39584 15.84 MB/sec warmup 73 sec latency 43.734 ms 2024-09-17T10:47:22.777 INFO:teuthology.orchestra.run.smithi006.stdout: 5 39969 15.70 MB/sec warmup 74 sec latency 47.813 ms 2024-09-17T10:47:22.912 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:22 smithi006 ceph-mon[23740]: pgmap v277: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.8 MiB/s wr, 714 op/s 2024-09-17T10:47:23.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:22 smithi149 ceph-mon[26044]: pgmap v277: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.8 MiB/s wr, 714 op/s 2024-09-17T10:47:23.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 40413 15.68 MB/sec warmup 75 sec latency 32.242 ms 2024-09-17T10:47:24.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 41024 15.59 MB/sec warmup 76 sec latency 35.830 ms 2024-09-17T10:47:24.945 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:24 smithi006 ceph-mon[23740]: pgmap v278: 97 pgs: 97 active+clean; 251 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 735 op/s 2024-09-17T10:47:25.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:24 smithi149 ceph-mon[26044]: pgmap v278: 97 pgs: 97 active+clean; 251 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 735 op/s 2024-09-17T10:47:25.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 41430 15.47 MB/sec warmup 77 sec latency 83.738 ms 2024-09-17T10:47:26.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 41708 15.41 MB/sec warmup 78 sec latency 49.172 ms 2024-09-17T10:47:27.019 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:26 smithi006 ceph-mon[23740]: pgmap v279: 97 pgs: 97 active+clean; 252 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 621 op/s 2024-09-17T10:47:27.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:26 smithi149 ceph-mon[26044]: pgmap v279: 97 pgs: 97 active+clean; 252 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 621 op/s 2024-09-17T10:47:27.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 42120 15.46 MB/sec warmup 79 sec latency 67.909 ms 2024-09-17T10:47:28.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 42636 15.54 MB/sec warmup 80 sec latency 34.246 ms 2024-09-17T10:47:29.018 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:28 smithi006 ceph-mon[23740]: pgmap v280: 97 pgs: 97 active+clean; 249 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 641 op/s 2024-09-17T10:47:29.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:28 smithi149 ceph-mon[26044]: pgmap v280: 97 pgs: 97 active+clean; 249 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 641 op/s 2024-09-17T10:47:29.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 43002 15.46 MB/sec warmup 81 sec latency 54.021 ms 2024-09-17T10:47:30.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 43333 15.30 MB/sec warmup 82 sec latency 61.851 ms 2024-09-17T10:47:31.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:30 smithi006 ceph-mon[23740]: pgmap v281: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.2 MiB/s wr, 638 op/s 2024-09-17T10:47:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:30 smithi149 ceph-mon[26044]: pgmap v281: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.2 MiB/s wr, 638 op/s 2024-09-17T10:47:31.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 43617 15.17 MB/sec warmup 83 sec latency 63.112 ms 2024-09-17T10:47:32.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 43999 15.14 MB/sec warmup 84 sec latency 47.722 ms 2024-09-17T10:47:33.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:32 smithi006 ceph-mon[23740]: pgmap v282: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.6 MiB/s wr, 609 op/s 2024-09-17T10:47:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:32 smithi149 ceph-mon[26044]: pgmap v282: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.6 MiB/s wr, 609 op/s 2024-09-17T10:47:33.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 44582 15.08 MB/sec warmup 85 sec latency 40.924 ms 2024-09-17T10:47:34.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 44873 14.95 MB/sec warmup 86 sec latency 96.771 ms 2024-09-17T10:47:35.145 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:34 smithi006 ceph-mon[23740]: pgmap v283: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 601 op/s 2024-09-17T10:47:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:34 smithi149 ceph-mon[26044]: pgmap v283: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 601 op/s 2024-09-17T10:47:35.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 45231 14.94 MB/sec warmup 87 sec latency 68.138 ms 2024-09-17T10:47:36.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 45654 14.94 MB/sec warmup 88 sec latency 62.848 ms 2024-09-17T10:47:36.925 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:36 smithi006 ceph-mon[23740]: pgmap v284: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 640 op/s 2024-09-17T10:47:37.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:36 smithi149 ceph-mon[26044]: pgmap v284: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 640 op/s 2024-09-17T10:47:37.780 INFO:teuthology.orchestra.run.smithi006.stdout: 5 46201 15.04 MB/sec warmup 89 sec latency 64.645 ms 2024-09-17T10:47:38.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 46476 14.95 MB/sec warmup 90 sec latency 40.388 ms 2024-09-17T10:47:38.959 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:38 smithi006 ceph-mon[23740]: pgmap v285: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-09-17T10:47:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:38 smithi149 ceph-mon[26044]: pgmap v285: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-09-17T10:47:39.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 46707 14.82 MB/sec warmup 91 sec latency 86.275 ms 2024-09-17T10:47:40.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 46949 14.68 MB/sec warmup 92 sec latency 130.548 ms 2024-09-17T10:47:41.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:40 smithi006 ceph-mon[23740]: pgmap v286: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 502 op/s 2024-09-17T10:47:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:40 smithi149 ceph-mon[26044]: pgmap v286: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 502 op/s 2024-09-17T10:47:41.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 47291 14.64 MB/sec warmup 93 sec latency 96.538 ms 2024-09-17T10:47:42.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 47669 14.55 MB/sec warmup 94 sec latency 75.350 ms 2024-09-17T10:47:43.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:42 smithi006 ceph-mon[23740]: pgmap v287: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.9 MiB/s wr, 538 op/s 2024-09-17T10:47:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:42 smithi149 ceph-mon[26044]: pgmap v287: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.9 MiB/s wr, 538 op/s 2024-09-17T10:47:43.778 INFO:teuthology.orchestra.run.smithi006.stdout: 5 48020 14.45 MB/sec warmup 95 sec latency 341.459 ms 2024-09-17T10:47:44.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 48659 14.45 MB/sec warmup 96 sec latency 32.550 ms 2024-09-17T10:47:45.020 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:44 smithi006 ceph-mon[23740]: pgmap v288: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 612 op/s 2024-09-17T10:47:45.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:44 smithi149 ceph-mon[26044]: pgmap v288: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 612 op/s 2024-09-17T10:47:45.781 INFO:teuthology.orchestra.run.smithi006.stdout: 5 49198 14.53 MB/sec warmup 97 sec latency 43.057 ms 2024-09-17T10:47:46.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 49716 14.63 MB/sec warmup 98 sec latency 32.751 ms 2024-09-17T10:47:47.088 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:46 smithi006 ceph-mon[23740]: pgmap v289: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.7 MiB/s wr, 626 op/s 2024-09-17T10:47:47.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:46 smithi149 ceph-mon[26044]: pgmap v289: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.7 MiB/s wr, 626 op/s 2024-09-17T10:47:47.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 50193 14.58 MB/sec warmup 99 sec latency 32.831 ms 2024-09-17T10:47:47.909 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:47:48.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 50774 14.52 MB/sec warmup 100 sec latency 14.872 ms 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (6m) 102s ago 8m 22.3M - ba2b418f427c b74431e6f177 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (8m) 102s ago 8m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (7m) 104s ago 7m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (6m) 102s ago 7m 55.3M - 8.3.5 dad864ee21e9 fcaa29cab62f 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (2m) 102s ago 2m 14.7M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (2m) 104s ago 2m 12.0M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:9283 running (9m) 102s ago 9m 459M - 17.2.0 e1d6a67b021e 8caa6368de72 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283 running (7m) 104s ago 7m 412M - 17.2.0 e1d6a67b021e 34dc4cd55cb3 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (9m) 102s ago 9m 49.5M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:47:48.828 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (7m) 104s ago 7m 36.5M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (113s) 102s ago 112s 58.2M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (107s) 104s ago 107s 50.7M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (7m) 102s ago 7m 15.8M - 1dbe0e931976 19b622692f8e 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (7m) 104s ago 7m 16.8M - 1dbe0e931976 b3c75ec0fcd4 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (6m) 102s ago 6m 43.7M 3217M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (6m) 102s ago 6m 43.0M 3217M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (5m) 102s ago 5m 39.4M 3217M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (5m) 102s ago 5m 41.4M 3217M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (4m) 104s ago 4m 39.8M 1165M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (4m) 104s ago 4m 40.1M 1165M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (4m) 104s ago 4m 41.7M 1165M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:47:48.829 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (3m) 104s ago 3m 39.3M 1165M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:47:48.830 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (7m) 102s ago 7m 62.9M - 514e6a882f6e f5cf8e877b8a 2024-09-17T10:47:49.103 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:48 smithi006 ceph-mon[23740]: pgmap v290: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 560 op/s 2024-09-17T10:47:49.103 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:48 smithi006 ceph-mon[23740]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:49.252 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:48 smithi149 ceph-mon[26044]: pgmap v290: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 560 op/s 2024-09-17T10:47:49.252 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:48 smithi149 ceph-mon[26044]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:49.438 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:47:49.439 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:47:49.440 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:47:49.440 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T10:47:49.440 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:47:49.440 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:47:49.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 51310 14.52 MB/sec warmup 101 sec latency 80.509 ms 2024-09-17T10:47:49.909 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:47:49.909 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:47:49.909 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:47:49.909 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-09-17T10:47:49.909 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "0/23 daemons upgraded", 2024-09-17T10:47:49.910 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mgr daemons" 2024-09-17T10:47:49.910 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:47:50.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:49 smithi006 ceph-mon[23740]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:50.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:49 smithi006 ceph-mon[23740]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:50.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:47:50.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:47:50.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:47:50.086 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:49 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:47:50.086 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:49 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1165907305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:47:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:49 smithi149 ceph-mon[26044]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:49 smithi149 ceph-mon[26044]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:47:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:47:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:47:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:49 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:47:50.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:49 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1165907305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:47:50.485 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:47:50.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 51795 14.45 MB/sec warmup 102 sec latency 73.308 ms 2024-09-17T10:47:51.042 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:50 smithi149 ceph-mon[26044]: Upgrade: Updating mgr.smithi149.ijckag 2024-09-17T10:47:51.042 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:50 smithi149 ceph-mon[26044]: Deploying daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:47:51.042 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:50 smithi149 ceph-mon[26044]: pgmap v291: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.6 MiB/s wr, 537 op/s 2024-09-17T10:47:51.042 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:50 smithi149 ceph-mon[26044]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:51.042 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:50 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/870652843' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:47:51.117 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:50 smithi006 ceph-mon[23740]: Upgrade: Updating mgr.smithi149.ijckag 2024-09-17T10:47:51.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:50 smithi006 ceph-mon[23740]: Deploying daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:47:51.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:50 smithi006 ceph-mon[23740]: pgmap v291: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.6 MiB/s wr, 537 op/s 2024-09-17T10:47:51.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:50 smithi006 ceph-mon[23740]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:47:51.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:50 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/870652843' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:47:51.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 52560 14.56 MB/sec warmup 103 sec latency 22.395 ms 2024-09-17T10:47:52.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 53287 14.70 MB/sec warmup 104 sec latency 38.566 ms 2024-09-17T10:47:53.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:52 smithi006 ceph-mon[23740]: pgmap v292: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-09-17T10:47:53.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:47:53.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:47:53.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:47:53.136 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:52 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:47:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:52 smithi149 ceph-mon[26044]: pgmap v292: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-09-17T10:47:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:47:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:47:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:47:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:52 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:47:53.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 53752 14.66 MB/sec warmup 105 sec latency 30.225 ms 2024-09-17T10:47:54.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 53984 14.54 MB/sec warmup 106 sec latency 166.683 ms 2024-09-17T10:47:55.166 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:54 smithi006 ceph-mon[23740]: pgmap v293: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 656 op/s 2024-09-17T10:47:55.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:54 smithi149 ceph-mon[26044]: pgmap v293: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 656 op/s 2024-09-17T10:47:55.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 54288 14.46 MB/sec warmup 107 sec latency 68.766 ms 2024-09-17T10:47:56.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 54785 14.46 MB/sec warmup 108 sec latency 42.515 ms 2024-09-17T10:47:57.198 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:56 smithi006 ceph-mon[23740]: pgmap v294: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.0 MiB/s wr, 663 op/s 2024-09-17T10:47:57.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:56 smithi149 ceph-mon[26044]: pgmap v294: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.0 MiB/s wr, 663 op/s 2024-09-17T10:47:57.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 55412 14.41 MB/sec warmup 109 sec latency 58.286 ms 2024-09-17T10:47:58.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 55663 14.32 MB/sec warmup 110 sec latency 85.284 ms 2024-09-17T10:47:58.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:47:58 smithi006 ceph-mon[23740]: pgmap v295: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 622 op/s 2024-09-17T10:47:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:47:58 smithi149 ceph-mon[26044]: pgmap v295: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 622 op/s 2024-09-17T10:47:59.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 55995 14.33 MB/sec warmup 111 sec latency 63.066 ms 2024-09-17T10:48:00.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:00 smithi006 ceph-mon[23740]: Standby manager daemon smithi149.ijckag restarted 2024-09-17T10:48:00.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:00 smithi006 ceph-mon[23740]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:48:00.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:00 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:48:00.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:00 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:48:00.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:00 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:48:00.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:00 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:48:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:00 smithi149 ceph-mon[26044]: Standby manager daemon smithi149.ijckag restarted 2024-09-17T10:48:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:00 smithi149 ceph-mon[26044]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:48:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:00 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:48:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:00 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:48:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:00 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:48:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:00 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:48:00.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 56291 14.29 MB/sec warmup 112 sec latency 208.230 ms 2024-09-17T10:48:01.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:01 smithi006 ceph-mon[23740]: pgmap v296: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 599 op/s 2024-09-17T10:48:01.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:01 smithi006 ceph-mon[23740]: mgrmap e22: smithi006.plhvdm(active, since 7m), standbys: smithi149.ijckag 2024-09-17T10:48:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:01 smithi149 ceph-mon[26044]: pgmap v296: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 599 op/s 2024-09-17T10:48:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:01 smithi149 ceph-mon[26044]: mgrmap e22: smithi006.plhvdm(active, since 7m), standbys: smithi149.ijckag 2024-09-17T10:48:01.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 56648 14.32 MB/sec warmup 113 sec latency 59.205 ms 2024-09-17T10:48:02.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:48:02.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:02 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:48:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:48:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:02 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:48:02.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 56964 14.29 MB/sec warmup 114 sec latency 90.476 ms 2024-09-17T10:48:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:03 smithi149 ceph-mon[26044]: pgmap v297: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.2 MiB/s wr, 719 op/s 2024-09-17T10:48:03.357 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:03 smithi006 ceph-mon[23740]: pgmap v297: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.2 MiB/s wr, 719 op/s 2024-09-17T10:48:03.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 57267 14.21 MB/sec warmup 115 sec latency 44.724 ms 2024-09-17T10:48:04.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 57679 14.13 MB/sec warmup 116 sec latency 36.932 ms 2024-09-17T10:48:05.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:05 smithi006 ceph-mon[23740]: pgmap v298: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.7 MiB/s wr, 752 op/s 2024-09-17T10:48:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:05 smithi149 ceph-mon[26044]: pgmap v298: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.7 MiB/s wr, 752 op/s 2024-09-17T10:48:05.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 58106 14.13 MB/sec warmup 117 sec latency 53.230 ms 2024-09-17T10:48:06.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:06 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:06.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:06 smithi006 ceph-mon[23740]: pgmap v299: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.7 MiB/s wr, 643 op/s 2024-09-17T10:48:06.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:06 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:06.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 58786 14.10 MB/sec warmup 118 sec latency 41.556 ms 2024-09-17T10:48:06.866 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:06 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:06.866 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:06 smithi149 ceph-mon[26044]: pgmap v299: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.7 MiB/s wr, 643 op/s 2024-09-17T10:48:06.866 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:06 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:07.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 59575 14.17 MB/sec warmup 119 sec latency 23.603 ms 2024-09-17T10:48:08.968 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:08 smithi006 ceph-mon[23740]: pgmap v300: 97 pgs: 97 active+clean; 313 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 591 op/s 2024-09-17T10:48:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:08 smithi149 ceph-mon[26044]: pgmap v300: 97 pgs: 97 active+clean; 313 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 591 op/s 2024-09-17T10:48:09.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 60587 24.32 MB/sec execute 1 sec latency 41.854 ms 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: pgmap v301: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: Upgrade: Need to upgrade myself (mgr.smithi006.plhvdm) 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: Failing over to other MGR 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr fail", "who": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:48:10.739 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:10 smithi006 ceph-mon[23740]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T10:48:10.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 60934 13.91 MB/sec execute 2 sec latency 46.237 ms 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: pgmap v301: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: Upgrade: Need to upgrade myself (mgr.smithi006.plhvdm) 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: Failing over to other MGR 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr fail", "who": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:48:10.933 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:10 smithi149 ceph-mon[26044]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T10:48:11.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:48:11.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:48:11.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"}]: dispatch 2024-09-17T10:48:11.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"}]: dispatch 2024-09-17T10:48:11.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr metadata", "who": "smithi149.ijckag", "id": "smithi149.ijckag"}]: dispatch 2024-09-17T10:48:11.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:48:11.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:48:11.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "mgr fail", "who": "smithi006.plhvdm"}]': finished 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: mgrmap e23: smithi149.ijckag(active, starting, since 0.993738s) 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T10:48:11.767 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:11 smithi006 ceph-mon[23740]: Manager daemon smithi149.ijckag is now available 2024-09-17T10:48:11.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 61260 10.67 MB/sec execute 3 sec latency 31.256 ms 2024-09-17T10:48:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr metadata", "who": "smithi149.ijckag", "id": "smithi149.ijckag"}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.14200 172.21.15.6:0/2730878143' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "mgr fail", "who": "smithi006.plhvdm"}]': finished 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: mgrmap e23: smithi149.ijckag(active, starting, since 0.993738s) 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:48:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:48:11.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:48:11.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:48:11.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:48:11.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T10:48:11.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T10:48:11.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T10:48:11.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:11 smithi149 ceph-mon[26044]: Manager daemon smithi149.ijckag is now available 2024-09-17T10:48:12.781 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: mgrmap e24: smithi149.ijckag(active, since 2s) 2024-09-17T10:48:12.781 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.781 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.781 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.781 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: Checking for cert/key for grafana.smithi006 2024-09-17T10:48:12.781 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.781 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.781 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.ijckag/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.ijckag/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.ijckag/trash_purge_schedule"}]: dispatch 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.ijckag/trash_purge_schedule"}]: dispatch 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: Deploying cephadm binary to smithi006 2024-09-17T10:48:12.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:12 smithi006 ceph-mon[23740]: Deploying cephadm binary to smithi149 2024-09-17T10:48:12.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 61747 11.39 MB/sec execute 4 sec latency 31.352 ms 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: mgrmap e24: smithi149.ijckag(active, since 2s) 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: Checking for cert/key for grafana.smithi006 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:48:12.938 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:48:12.939 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.ijckag/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:48:12.939 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.ijckag/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:48:12.939 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.ijckag/trash_purge_schedule"}]: dispatch 2024-09-17T10:48:12.939 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.ijckag/trash_purge_schedule"}]: dispatch 2024-09-17T10:48:12.939 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: Deploying cephadm binary to smithi006 2024-09-17T10:48:12.939 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:12 smithi149 ceph-mon[26044]: Deploying cephadm binary to smithi149 2024-09-17T10:48:13.782 INFO:teuthology.orchestra.run.smithi006.stdout: 5 62285 11.15 MB/sec execute 5 sec latency 42.572 ms 2024-09-17T10:48:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:13 smithi006 ceph-mon[23740]: pgmap v4: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:48:14.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:13 smithi149 ceph-mon[26044]: pgmap v4: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:48:14.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 62711 10.35 MB/sec execute 6 sec latency 33.906 ms 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:14 smithi006 ceph-mon[23740]: mgrmap e25: smithi149.ijckag(active, since 4s) 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:14 smithi006 ceph-mon[23740]: [17/Sep/2024:10:48:14] ENGINE Bus STARTING 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:14 smithi006 ceph-mon[23740]: [17/Sep/2024:10:48:14] ENGINE Serving on https://172.21.15.149:7150 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:14 smithi006 ceph-mon[23740]: [17/Sep/2024:10:48:14] ENGINE Error in HTTPServer.serve 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Traceback (most recent call last): 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._connections.run(self.expiration_interval) 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._run(expiration_interval) 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T10:48:14.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: s = self.context.wrap_socket( 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: return self.sslsocket_class._create( 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self.do_handshake() 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._sslobj.do_handshake() 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T10:48:14.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:14 smithi006 ceph-mon[23740]: [17/Sep/2024:10:48:14] ENGINE Serving on http://172.21.15.149:8765 2024-09-17T10:48:14.816 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:14 smithi006 ceph-mon[23740]: [17/Sep/2024:10:48:14] ENGINE Bus STARTED 2024-09-17T10:48:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:14 smithi149 ceph-mon[26044]: mgrmap e25: smithi149.ijckag(active, since 4s) 2024-09-17T10:48:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:14 smithi149 ceph-mon[26044]: [17/Sep/2024:10:48:14] ENGINE Bus STARTING 2024-09-17T10:48:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:14 smithi149 ceph-mon[26044]: [17/Sep/2024:10:48:14] ENGINE Serving on https://172.21.15.149:7150 2024-09-17T10:48:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:14 smithi149 ceph-mon[26044]: [17/Sep/2024:10:48:14] ENGINE Error in HTTPServer.serve 2024-09-17T10:48:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Traceback (most recent call last): 2024-09-17T10:48:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T10:48:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._connections.run(self.expiration_interval) 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._run(expiration_interval) 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: s = self.context.wrap_socket( 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: return self.sslsocket_class._create( 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self.do_handshake() 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T10:48:15.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._sslobj.do_handshake() 2024-09-17T10:48:15.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T10:48:15.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:14 smithi149 ceph-mon[26044]: [17/Sep/2024:10:48:14] ENGINE Serving on http://172.21.15.149:8765 2024-09-17T10:48:15.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:14 smithi149 ceph-mon[26044]: [17/Sep/2024:10:48:14] ENGINE Bus STARTED 2024-09-17T10:48:15.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 63216 12.10 MB/sec execute 7 sec latency 32.215 ms 2024-09-17T10:48:15.851 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:15 smithi006 ceph-mon[23740]: pgmap v5: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:48:15.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:15 smithi149 ceph-mon[26044]: pgmap v5: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:48:16.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 63547 11.86 MB/sec execute 8 sec latency 155.381 ms 2024-09-17T10:48:16.898 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:16 smithi006 ceph-mon[23740]: mgrmap e26: smithi149.ijckag(active, since 6s) 2024-09-17T10:48:16.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:16 smithi149 ceph-mon[26044]: mgrmap e26: smithi149.ijckag(active, since 6s) 2024-09-17T10:48:17.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 63958 12.63 MB/sec execute 9 sec latency 60.741 ms 2024-09-17T10:48:17.910 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:17 smithi006 ceph-mon[23740]: pgmap v6: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:48:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:17 smithi149 ceph-mon[26044]: pgmap v6: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:48:18.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 64214 12.09 MB/sec execute 10 sec latency 42.687 ms 2024-09-17T10:48:19.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 64500 11.25 MB/sec execute 11 sec latency 69.655 ms 2024-09-17T10:48:19.926 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:19 smithi006 ceph-mon[23740]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 4.7 MiB/s wr, 590 op/s 2024-09-17T10:48:19.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:19 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:19.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:19 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:20.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:19 smithi149 ceph-mon[26044]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 4.7 MiB/s wr, 590 op/s 2024-09-17T10:48:20.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:19 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:20.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:19 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:20.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 64837 10.73 MB/sec execute 12 sec latency 43.299 ms 2024-09-17T10:48:20.946 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:20 smithi006 ceph-mon[23740]: Standby manager daemon smithi006.plhvdm started 2024-09-17T10:48:20.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:20 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.6:0/1515633613' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.plhvdm/crt"}]: dispatch 2024-09-17T10:48:20.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:20 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.6:0/1515633613' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:48:20.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:20 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.6:0/1515633613' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.plhvdm/key"}]: dispatch 2024-09-17T10:48:20.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:20 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.6:0/1515633613' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:48:20.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:20 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:20.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:20 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:20.999 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:48:21.183 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:20 smithi149 ceph-mon[26044]: Standby manager daemon smithi006.plhvdm started 2024-09-17T10:48:21.183 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:20 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.6:0/1515633613' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.plhvdm/crt"}]: dispatch 2024-09-17T10:48:21.183 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:20 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.6:0/1515633613' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:48:21.183 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:20 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.6:0/1515633613' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.plhvdm/key"}]: dispatch 2024-09-17T10:48:21.183 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:20 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.6:0/1515633613' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:48:21.183 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:20 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:21.183 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:20 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:21.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 65273 10.85 MB/sec execute 13 sec latency 42.502 ms 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (7m) 0s ago 8m 26.0M - 0.23.0 ba2b418f427c b74431e6f177 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (8m) 0s ago 8m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (8m) 2s ago 8m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (7m) 0s ago 8m 54.9M - dad864ee21e9 fcaa29cab62f 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (2m) 0s ago 2m 35.1M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (2m) 2s ago 2m 12.8M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:9283 running (9m) 0s ago 9m 154M - 17.2.0 e1d6a67b021e 8caa6368de72 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283 running (29s) 2s ago 8m 496M - 19.3.0-4937-g74984932 6c1fcab85602 053d414e8a22 2024-09-17T10:48:22.004 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (10m) 0s ago 10m 53.0M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (8m) 2s ago 8m 38.8M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (2m) 0s ago 2m 243M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (2m) 2s ago 2m 77.3M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (8m) 0s ago 8m 17.2M - 1.3.1 1dbe0e931976 19b622692f8e 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (7m) 2s ago 7m 19.5M - 1.3.1 1dbe0e931976 b3c75ec0fcd4 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (7m) 0s ago 7m 282M 3217M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (6m) 0s ago 6m 203M 3217M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (6m) 0s ago 6m 192M 3217M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (5m) 0s ago 5m 222M 3217M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (5m) 2s ago 5m 241M 1165M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (5m) 2s ago 5m 274M 1165M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (4m) 2s ago 4m 246M 1165M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:48:22.005 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (4m) 2s ago 4m 329M 1165M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:48:22.006 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (7m) 0s ago 7m 67.3M - 2.33.4 514e6a882f6e f5cf8e877b8a 2024-09-17T10:48:22.098 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:21 smithi006 ceph-mon[23740]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.6 MiB/s wr, 458 op/s 2024-09-17T10:48:22.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:21 smithi006 ceph-mon[23740]: mgrmap e27: smithi149.ijckag(active, since 11s), standbys: smithi006.plhvdm 2024-09-17T10:48:22.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:21 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:48:22.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:21 smithi006 ceph-mon[23740]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:22.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:21 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:22.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:21 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:22.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:21 smithi006 ceph-mon[23740]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:22.156 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:21 smithi149 ceph-mon[26044]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.6 MiB/s wr, 458 op/s 2024-09-17T10:48:22.156 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:21 smithi149 ceph-mon[26044]: mgrmap e27: smithi149.ijckag(active, since 11s), standbys: smithi006.plhvdm 2024-09-17T10:48:22.156 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:21 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:48:22.156 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:21 smithi149 ceph-mon[26044]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:22.156 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:21 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:22.156 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:21 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:22.156 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:21 smithi149 ceph-mon[26044]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:22.613 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:48:22.613 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:48:22.613 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:48:22.613 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T10:48:22.614 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:48:22.615 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:48:22.615 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:48:22.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 65729 10.57 MB/sec execute 14 sec latency 51.653 ms 2024-09-17T10:48:23.052 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:48:23.052 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:48:23.052 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:48:23.052 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:48:23.052 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-09-17T10:48:23.052 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T10:48:23.052 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-09-17T10:48:23.053 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:48:23.053 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:48:23.382 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: Adjusting osd_memory_target on smithi006 to 1937M 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:23.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:23 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1950803724' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:48:23.641 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:48:23.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:23.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:23.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:23.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.684 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.684 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.684 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:23.684 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: Adjusting osd_memory_target on smithi006 to 1937M 2024-09-17T10:48:23.684 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:23.684 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:23 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1950803724' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:48:23.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 66060 10.34 MB/sec execute 15 sec latency 72.540 ms 2024-09-17T10:48:24.446 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: pgmap v9: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.4 MiB/s wr, 611 op/s 2024-09-17T10:48:24.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:24.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/470284904' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:48:24.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:24.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:24.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:24.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:24.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd='[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]': finished 2024-09-17T10:48:24.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:48:24.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:24 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:48:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: pgmap v9: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.4 MiB/s wr, 611 op/s 2024-09-17T10:48:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/470284904' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:48:24.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:24.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:24.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:24.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:48:24.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd='[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]': finished 2024-09-17T10:48:24.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:48:24.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:24 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:48:24.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 66417 10.24 MB/sec execute 16 sec latency 43.603 ms 2024-09-17T10:48:25.493 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:48:25.493 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: Updating smithi149:/etc/ceph/ceph.conf 2024-09-17T10:48:25.493 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:48:25.493 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:48:25.493 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:48:25.493 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:25 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:48:25.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: Updating smithi149:/etc/ceph/ceph.conf 2024-09-17T10:48:25.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:48:25.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:48:25.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:25 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:25.783 INFO:teuthology.orchestra.run.smithi006.stdout: 5 66793 10.63 MB/sec execute 17 sec latency 61.628 ms 2024-09-17T10:48:26.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:26 smithi006 ceph-mon[23740]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:48:26.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:26 smithi006 ceph-mon[23740]: pgmap v10: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.9 MiB/s wr, 558 op/s 2024-09-17T10:48:26.506 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:26 smithi006 ceph-mon[23740]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:48:26.506 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:26 smithi006 ceph-mon[23740]: Reconfiguring alertmanager.smithi006 (dependencies changed)... 2024-09-17T10:48:26.506 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:26 smithi006 ceph-mon[23740]: Deploying daemon alertmanager.smithi006 on smithi006 2024-09-17T10:48:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:26 smithi149 ceph-mon[26044]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:48:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:26 smithi149 ceph-mon[26044]: pgmap v10: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.9 MiB/s wr, 558 op/s 2024-09-17T10:48:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:26 smithi149 ceph-mon[26044]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:48:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:26 smithi149 ceph-mon[26044]: Reconfiguring alertmanager.smithi006 (dependencies changed)... 2024-09-17T10:48:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:26 smithi149 ceph-mon[26044]: Deploying daemon alertmanager.smithi006 on smithi006 2024-09-17T10:48:26.784 INFO:teuthology.orchestra.run.smithi006.stdout: 5 67464 11.54 MB/sec execute 18 sec latency 50.644 ms 2024-09-17T10:48:27.784 INFO:teuthology.orchestra.run.smithi006.stdout: 5 67927 11.50 MB/sec execute 19 sec latency 22.358 ms 2024-09-17T10:48:28.012 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:27 smithi006 ceph-mon[23740]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.9 MiB/s wr, 558 op/s 2024-09-17T10:48:28.013 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:27 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:48:28.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:27 smithi149 ceph-mon[26044]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.9 MiB/s wr, 558 op/s 2024-09-17T10:48:28.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:27 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:48:28.784 INFO:teuthology.orchestra.run.smithi006.stdout: 5 68520 11.44 MB/sec execute 20 sec latency 77.511 ms 2024-09-17T10:48:29.784 INFO:teuthology.orchestra.run.smithi006.stdout: 5 69223 11.58 MB/sec execute 21 sec latency 23.123 ms 2024-09-17T10:48:30.021 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:29 smithi006 ceph-mon[23740]: pgmap v12: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 7.5 MiB/s wr, 805 op/s 2024-09-17T10:48:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:29 smithi149 ceph-mon[26044]: pgmap v12: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 7.5 MiB/s wr, 805 op/s 2024-09-17T10:48:30.784 INFO:teuthology.orchestra.run.smithi006.stdout: 5 69893 11.72 MB/sec execute 22 sec latency 30.171 ms 2024-09-17T10:48:31.784 INFO:teuthology.orchestra.run.smithi006.stdout: 5 70619 12.41 MB/sec execute 23 sec latency 21.415 ms 2024-09-17T10:48:31.916 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:31 smithi006 ceph-mon[23740]: pgmap v13: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-17T10:48:32.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:31 smithi149 ceph-mon[26044]: pgmap v13: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-17T10:48:32.784 INFO:teuthology.orchestra.run.smithi006.stdout: 5 71146 12.84 MB/sec execute 24 sec latency 28.632 ms 2024-09-17T10:48:33.785 INFO:teuthology.orchestra.run.smithi006.stdout: 5 71575 12.59 MB/sec execute 25 sec latency 33.846 ms 2024-09-17T10:48:34.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:33 smithi006 ceph-mon[23740]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 834 op/s 2024-09-17T10:48:34.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:33 smithi149 ceph-mon[26044]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 834 op/s 2024-09-17T10:48:34.785 INFO:teuthology.orchestra.run.smithi006.stdout: 5 72195 12.64 MB/sec execute 26 sec latency 25.564 ms 2024-09-17T10:48:35.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 72898 12.58 MB/sec execute 27 sec latency 20.496 ms 2024-09-17T10:48:36.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:35 smithi006 ceph-mon[23740]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 618 op/s 2024-09-17T10:48:36.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:35 smithi149 ceph-mon[26044]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 618 op/s 2024-09-17T10:48:36.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 73436 12.62 MB/sec execute 28 sec latency 35.698 ms 2024-09-17T10:48:37.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 73971 12.87 MB/sec execute 29 sec latency 32.584 ms 2024-09-17T10:48:38.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:37 smithi006 ceph-mon[23740]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 618 op/s 2024-09-17T10:48:38.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:37 smithi149 ceph-mon[26044]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 618 op/s 2024-09-17T10:48:38.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 74557 13.29 MB/sec execute 30 sec latency 44.833 ms 2024-09-17T10:48:39.793 INFO:teuthology.orchestra.run.smithi006.stdout: 5 74947 13.18 MB/sec execute 31 sec latency 70.033 ms 2024-09-17T10:48:40.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:39 smithi006 ceph-mon[23740]: pgmap v17: 97 pgs: 97 active+clean; 319 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 945 op/s 2024-09-17T10:48:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:39 smithi149 ceph-mon[26044]: pgmap v17: 97 pgs: 97 active+clean; 319 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 945 op/s 2024-09-17T10:48:40.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 75393 12.93 MB/sec execute 32 sec latency 47.329 ms 2024-09-17T10:48:41.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 75974 13.06 MB/sec execute 33 sec latency 43.066 ms 2024-09-17T10:48:42.137 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:41 smithi006 ceph-mon[23740]: pgmap v18: 97 pgs: 97 active+clean; 319 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 698 op/s 2024-09-17T10:48:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:41 smithi149 ceph-mon[26044]: pgmap v18: 97 pgs: 97 active+clean; 319 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 698 op/s 2024-09-17T10:48:42.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 76542 12.82 MB/sec execute 34 sec latency 28.571 ms 2024-09-17T10:48:42.931 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:42 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:48:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:42 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:48:43.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 77005 12.86 MB/sec execute 35 sec latency 30.816 ms 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:43 smithi006 ceph-mon[23740]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 994 op/s 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: Reconfiguring grafana.smithi006 (dependencies changed)... 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T10:48:44.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:44 smithi006 ceph-mon[23740]: Reconfiguring daemon grafana.smithi006 on smithi006 2024-09-17T10:48:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 994 op/s 2024-09-17T10:48:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:44.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: Reconfiguring grafana.smithi006 (dependencies changed)... 2024-09-17T10:48:44.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T10:48:44.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:44.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:44.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T10:48:44.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T10:48:44.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:44 smithi149 ceph-mon[26044]: Reconfiguring daemon grafana.smithi006 on smithi006 2024-09-17T10:48:44.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 77708 13.21 MB/sec execute 36 sec latency 25.141 ms 2024-09-17T10:48:45.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 78148 13.37 MB/sec execute 37 sec latency 76.626 ms 2024-09-17T10:48:46.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:46 smithi006 ceph-mon[23740]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 622 op/s 2024-09-17T10:48:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:46 smithi149 ceph-mon[26044]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 622 op/s 2024-09-17T10:48:46.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 78514 13.28 MB/sec execute 38 sec latency 97.023 ms 2024-09-17T10:48:47.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 79083 13.18 MB/sec execute 39 sec latency 43.687 ms 2024-09-17T10:48:48.016 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:48 smithi006 ceph-mon[23740]: pgmap v21: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 622 op/s 2024-09-17T10:48:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:48 smithi149 ceph-mon[26044]: pgmap v21: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 622 op/s 2024-09-17T10:48:48.793 INFO:teuthology.orchestra.run.smithi006.stdout: 5 79336 12.94 MB/sec execute 40 sec latency 60.051 ms 2024-09-17T10:48:49.791 INFO:teuthology.orchestra.run.smithi006.stdout: 5 79715 12.92 MB/sec execute 41 sec latency 41.144 ms 2024-09-17T10:48:50.076 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:50 smithi006 ceph-mon[23740]: pgmap v22: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 924 op/s 2024-09-17T10:48:50.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:50 smithi149 ceph-mon[26044]: pgmap v22: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 924 op/s 2024-09-17T10:48:50.792 INFO:teuthology.orchestra.run.smithi006.stdout: 5 80154 12.77 MB/sec execute 42 sec latency 56.709 ms 2024-09-17T10:48:51.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 80600 12.69 MB/sec execute 43 sec latency 55.619 ms 2024-09-17T10:48:52.093 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:52 smithi006 ceph-mon[23740]: pgmap v23: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.7 MiB/s wr, 598 op/s 2024-09-17T10:48:52.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:52 smithi149 ceph-mon[26044]: pgmap v23: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.7 MiB/s wr, 598 op/s 2024-09-17T10:48:52.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 81099 12.88 MB/sec execute 44 sec latency 72.512 ms 2024-09-17T10:48:53.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 81584 13.04 MB/sec execute 45 sec latency 65.227 ms 2024-09-17T10:48:53.858 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:53 smithi006 ceph-mon[23740]: pgmap v24: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 855 op/s 2024-09-17T10:48:53.859 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:53 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:53.859 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:53 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:53.859 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:53 smithi006 ceph-mon[23740]: Reconfiguring node-exporter.smithi006 (dependencies changed)... 2024-09-17T10:48:53.859 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:53 smithi006 ceph-mon[23740]: Deploying daemon node-exporter.smithi006 on smithi006 2024-09-17T10:48:54.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:53 smithi149 ceph-mon[26044]: pgmap v24: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 855 op/s 2024-09-17T10:48:54.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:53 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:54.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:53 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:48:54.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:53 smithi149 ceph-mon[26044]: Reconfiguring node-exporter.smithi006 (dependencies changed)... 2024-09-17T10:48:54.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:53 smithi149 ceph-mon[26044]: Deploying daemon node-exporter.smithi006 on smithi006 2024-09-17T10:48:54.242 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:48:54.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 81986 13.02 MB/sec execute 46 sec latency 69.342 ms 2024-09-17T10:48:54.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:54 smithi006 ceph-mon[23740]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:54 smithi149 ceph-mon[26044]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:55.374 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 starting - - - - 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (9m) 33s ago 9m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (8m) 35s ago 8m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 starting - - - - 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (3m) 33s ago 3m 35.1M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (3m) 35s ago 3m 12.8M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:9283 running (10m) 33s ago 10m 154M - 17.2.0 e1d6a67b021e 8caa6368de72 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283 running (63s) 35s ago 8m 496M - 19.3.0-4937-g74984932 6c1fcab85602 053d414e8a22 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (10m) 33s ago 10m 53.0M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (8m) 35s ago 8m 38.8M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (2m) 33s ago 2m 243M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:48:55.375 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (2m) 35s ago 2m 77.3M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (9m) 33s ago 9m 17.2M - 1.3.1 1dbe0e931976 19b622692f8e 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (8m) 35s ago 8m 19.5M - 1.3.1 1dbe0e931976 b3c75ec0fcd4 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (7m) 33s ago 7m 282M 1937M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (7m) 33s ago 7m 203M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (6m) 33s ago 6m 192M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (6m) 33s ago 6m 222M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (6m) 35s ago 6m 241M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (5m) 35s ago 5m 274M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (5m) 35s ago 5m 246M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (4m) 35s ago 4m 329M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:48:55.376 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (8m) 33s ago 8m 67.3M - 2.33.4 514e6a882f6e f5cf8e877b8a 2024-09-17T10:48:55.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 82336 12.83 MB/sec execute 47 sec latency 35.824 ms 2024-09-17T10:48:55.881 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:55 smithi006 ceph-mon[23740]: pgmap v25: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.5 MiB/s wr, 559 op/s 2024-09-17T10:48:55.882 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:55 smithi006 ceph-mon[23740]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:55.882 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:55 smithi006 ceph-mon[23740]: from='client.24501 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:56.169 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:48:56.170 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:48:56.171 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:48:56.171 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:48:56.171 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T10:48:56.171 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:48:56.171 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:48:56.171 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:48:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:55 smithi149 ceph-mon[26044]: pgmap v25: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.5 MiB/s wr, 559 op/s 2024-09-17T10:48:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:55 smithi149 ceph-mon[26044]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:55 smithi149 ceph-mon[26044]: from='client.24501 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:56.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 82701 12.70 MB/sec execute 48 sec latency 37.486 ms 2024-09-17T10:48:56.832 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:48:56.832 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:48:56.832 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:48:56.833 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:48:56.833 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-09-17T10:48:56.833 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T10:48:56.833 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-09-17T10:48:56.833 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:48:56.833 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:48:56.891 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:56 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2814079494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:48:57.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:56 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/2814079494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:48:57.550 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:48:57.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 83212 12.73 MB/sec execute 49 sec latency 36.043 ms 2024-09-17T10:48:57.913 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:57 smithi006 ceph-mon[23740]: pgmap v26: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.5 MiB/s wr, 559 op/s 2024-09-17T10:48:57.913 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:57 smithi006 ceph-mon[23740]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:57.913 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:57 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:48:57.913 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:57 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/767452081' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:48:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:57 smithi149 ceph-mon[26044]: pgmap v26: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.5 MiB/s wr, 559 op/s 2024-09-17T10:48:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:57 smithi149 ceph-mon[26044]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:48:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:57 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:48:58.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:57 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/767452081' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:48:58.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 83699 12.63 MB/sec execute 50 sec latency 42.607 ms 2024-09-17T10:48:59.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 84103 12.57 MB/sec execute 51 sec latency 40.207 ms 2024-09-17T10:48:59.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:48:59 smithi006 ceph-mon[23740]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 805 op/s 2024-09-17T10:49:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:48:59 smithi149 ceph-mon[26044]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 805 op/s 2024-09-17T10:49:00.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 84389 12.50 MB/sec execute 52 sec latency 46.692 ms 2024-09-17T10:49:01.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 84895 12.68 MB/sec execute 53 sec latency 58.679 ms 2024-09-17T10:49:02.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:01 smithi006 ceph-mon[23740]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.1 MiB/s wr, 503 op/s 2024-09-17T10:49:02.043 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:01 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:02.043 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:01 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:02.043 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:01 smithi006 ceph-mon[23740]: Reconfiguring prometheus.smithi006 (dependencies changed)... 2024-09-17T10:49:02.043 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:01 smithi006 ceph-mon[23740]: Deploying daemon prometheus.smithi006 on smithi006 2024-09-17T10:49:02.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:01 smithi149 ceph-mon[26044]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.1 MiB/s wr, 503 op/s 2024-09-17T10:49:02.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:01 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:02.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:01 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:02.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:01 smithi149 ceph-mon[26044]: Reconfiguring prometheus.smithi006 (dependencies changed)... 2024-09-17T10:49:02.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:01 smithi149 ceph-mon[26044]: Deploying daemon prometheus.smithi006 on smithi006 2024-09-17T10:49:02.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 85262 12.72 MB/sec execute 54 sec latency 60.133 ms 2024-09-17T10:49:03.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 85558 12.63 MB/sec execute 55 sec latency 45.871 ms 2024-09-17T10:49:04.065 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:03 smithi006 ceph-mon[23740]: pgmap v29: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 761 op/s 2024-09-17T10:49:04.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:03 smithi149 ceph-mon[26044]: pgmap v29: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 761 op/s 2024-09-17T10:49:04.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 85895 12.48 MB/sec execute 56 sec latency 40.938 ms 2024-09-17T10:49:05.794 INFO:teuthology.orchestra.run.smithi006.stdout: 5 86265 12.39 MB/sec execute 57 sec latency 62.822 ms 2024-09-17T10:49:06.095 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:05 smithi006 ceph-mon[23740]: pgmap v30: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.5 MiB/s wr, 504 op/s 2024-09-17T10:49:06.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:05 smithi149 ceph-mon[26044]: pgmap v30: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.5 MiB/s wr, 504 op/s 2024-09-17T10:49:06.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 86654 12.35 MB/sec execute 58 sec latency 58.158 ms 2024-09-17T10:49:07.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 87019 12.24 MB/sec execute 59 sec latency 43.287 ms 2024-09-17T10:49:07.900 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:07 smithi006 ceph-mon[23740]: pgmap v31: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.5 MiB/s wr, 504 op/s 2024-09-17T10:49:08.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:07 smithi149 ceph-mon[26044]: pgmap v31: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.5 MiB/s wr, 504 op/s 2024-09-17T10:49:08.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 87522 12.23 MB/sec execute 60 sec latency 42.744 ms 2024-09-17T10:49:09.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 88080 12.36 MB/sec execute 61 sec latency 32.294 ms 2024-09-17T10:49:10.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:09 smithi006 ceph-mon[23740]: pgmap v32: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 721 op/s 2024-09-17T10:49:10.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:09 smithi149 ceph-mon[26044]: pgmap v32: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 721 op/s 2024-09-17T10:49:10.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 88490 12.40 MB/sec execute 62 sec latency 54.379 ms 2024-09-17T10:49:11.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 88909 12.45 MB/sec execute 63 sec latency 73.813 ms 2024-09-17T10:49:12.126 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:11 smithi006 ceph-mon[23740]: pgmap v33: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 475 op/s 2024-09-17T10:49:12.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:11 smithi149 ceph-mon[26044]: pgmap v33: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 475 op/s 2024-09-17T10:49:12.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 89212 12.39 MB/sec execute 64 sec latency 43.461 ms 2024-09-17T10:49:13.134 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:12 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:49:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:12 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:49:13.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 89606 12.29 MB/sec execute 65 sec latency 55.290 ms 2024-09-17T10:49:14.137 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:13 smithi006 ceph-mon[23740]: pgmap v34: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-09-17T10:49:14.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:13 smithi149 ceph-mon[26044]: pgmap v34: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-09-17T10:49:14.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 90103 12.23 MB/sec execute 66 sec latency 56.319 ms 2024-09-17T10:49:15.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 90532 12.21 MB/sec execute 67 sec latency 42.977 ms 2024-09-17T10:49:15.945 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:15 smithi006 ceph-mon[23740]: pgmap v35: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 475 op/s 2024-09-17T10:49:16.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:15 smithi149 ceph-mon[26044]: pgmap v35: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 475 op/s 2024-09-17T10:49:16.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 91139 12.23 MB/sec execute 68 sec latency 51.511 ms 2024-09-17T10:49:17.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 91788 12.40 MB/sec execute 69 sec latency 22.208 ms 2024-09-17T10:49:17.967 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:17 smithi006 ceph-mon[23740]: pgmap v36: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 475 op/s 2024-09-17T10:49:18.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:17 smithi149 ceph-mon[26044]: pgmap v36: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 475 op/s 2024-09-17T10:49:18.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 92394 12.55 MB/sec execute 70 sec latency 27.404 ms 2024-09-17T10:49:19.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 92687 12.49 MB/sec execute 71 sec latency 53.998 ms 2024-09-17T10:49:20.010 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:19 smithi006 ceph-mon[23740]: pgmap v37: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 751 op/s 2024-09-17T10:49:20.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:19 smithi149 ceph-mon[26044]: pgmap v37: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 751 op/s 2024-09-17T10:49:20.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 92966 12.36 MB/sec execute 72 sec latency 46.762 ms 2024-09-17T10:49:21.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 93318 12.29 MB/sec execute 73 sec latency 51.022 ms 2024-09-17T10:49:22.037 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:21 smithi006 ceph-mon[23740]: pgmap v38: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 534 op/s 2024-09-17T10:49:22.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:21 smithi149 ceph-mon[26044]: pgmap v38: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 534 op/s 2024-09-17T10:49:22.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 93958 12.32 MB/sec execute 74 sec latency 29.236 ms 2024-09-17T10:49:23.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 94576 12.34 MB/sec execute 75 sec latency 35.823 ms 2024-09-17T10:49:24.304 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:24 smithi006 ceph-mon[23740]: pgmap v39: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 799 op/s 2024-09-17T10:49:24.304 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:24 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:24.305 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:24 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:24.305 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:24 smithi006 ceph-mon[23740]: Reconfiguring node-exporter.smithi149 (dependencies changed)... 2024-09-17T10:49:24.305 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:24 smithi006 ceph-mon[23740]: Deploying daemon node-exporter.smithi149 on smithi149 2024-09-17T10:49:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:24 smithi149 ceph-mon[26044]: pgmap v39: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 799 op/s 2024-09-17T10:49:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:24 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:24 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:24 smithi149 ceph-mon[26044]: Reconfiguring node-exporter.smithi149 (dependencies changed)... 2024-09-17T10:49:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:24 smithi149 ceph-mon[26044]: Deploying daemon node-exporter.smithi149 on smithi149 2024-09-17T10:49:24.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 95145 12.42 MB/sec execute 76 sec latency 34.258 ms 2024-09-17T10:49:25.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 95659 12.52 MB/sec execute 77 sec latency 31.301 ms 2024-09-17T10:49:26.332 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:26 smithi006 ceph-mon[23740]: pgmap v40: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 541 op/s 2024-09-17T10:49:26.435 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:26 smithi149 ceph-mon[26044]: pgmap v40: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 541 op/s 2024-09-17T10:49:26.795 INFO:teuthology.orchestra.run.smithi006.stdout: 5 96062 12.52 MB/sec execute 78 sec latency 43.142 ms 2024-09-17T10:49:27.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 96350 12.45 MB/sec execute 79 sec latency 46.717 ms 2024-09-17T10:49:27.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:27 smithi149 ceph-mon[26044]: pgmap v41: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 541 op/s 2024-09-17T10:49:27.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:27 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:49:28.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:27 smithi006 ceph-mon[23740]: pgmap v41: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 541 op/s 2024-09-17T10:49:28.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:27 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:49:28.149 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:49:28.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 96761 12.38 MB/sec execute 80 sec latency 52.849 ms 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 starting - - - - 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (9m) 67s ago 9m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (9m) 69s ago 9m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 starting - - - - 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (3m) 67s ago 3m 35.1M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (3m) 69s ago 3m 12.8M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:9283 running (11m) 67s ago 11m 154M - 17.2.0 e1d6a67b021e 8caa6368de72 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283 running (97s) 69s ago 9m 496M - 19.3.0-4937-g74984932 6c1fcab85602 053d414e8a22 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (11m) 67s ago 11m 53.0M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:49:29.243 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (9m) 69s ago 9m 38.8M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (3m) 67s ago 3m 243M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (3m) 69s ago 3m 77.3M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 starting - - - - 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 starting - - - - 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (8m) 67s ago 8m 282M 1937M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (7m) 67s ago 7m 203M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (7m) 67s ago 7m 192M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (6m) 67s ago 6m 222M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (6m) 69s ago 6m 241M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (6m) 69s ago 6m 274M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:49:29.244 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (5m) 69s ago 5m 246M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:49:29.245 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (5m) 69s ago 5m 329M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:49:29.245 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 starting - - - - 2024-09-17T10:49:29.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 97262 12.37 MB/sec execute 81 sec latency 40.576 ms 2024-09-17T10:49:29.877 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='client.24517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:49:29.877 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.877 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi006.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi006.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:29.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:49:29.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:29 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='client.24517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:49:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi006.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi006.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:49:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:29 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:30.024 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:49:30.024 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:49:30.024 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:49:30.024 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:49:30.024 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:49:30.024 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T10:49:30.024 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:49:30.025 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:49:30.601 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:49:30.601 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:49:30.601 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:49:30.601 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:49:30.602 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-09-17T10:49:30.602 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T10:49:30.602 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mgr daemons", 2024-09-17T10:49:30.602 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:49:30.602 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:49:30.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 97661 12.31 MB/sec execute 82 sec latency 50.239 ms 2024-09-17T10:49:30.895 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T10:49:30.895 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T10:49:30.895 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:49:30.895 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:49:30.895 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi006.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T10:49:30.895 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:49:30.895 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi006.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T10:49:30.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: pgmap v42: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 861 op/s 2024-09-17T10:49:30.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:30.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: Upgrade: Need to upgrade myself (mgr.smithi149.ijckag) 2024-09-17T10:49:30.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: Upgrade: Need to upgrade myself (mgr.smithi149.ijckag) 2024-09-17T10:49:30.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='client.24525 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:49:30.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:30.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1712000681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:49:30.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:30 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T10:49:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T10:49:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi006.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi006.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: pgmap v42: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 861 op/s 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: Upgrade: Need to upgrade myself (mgr.smithi149.ijckag) 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: Upgrade: Need to upgrade myself (mgr.smithi149.ijckag) 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='client.24525 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1712000681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:49:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:30 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:31.318 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:49:31.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:31 smithi006 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:31.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:31 smithi006 ceph-mon[23740]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:49:31.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:31 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/2754469747' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:49:31.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 98129 12.30 MB/sec execute 83 sec latency 47.869 ms 2024-09-17T10:49:31.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:31 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:49:31.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:31 smithi149 ceph-mon[26044]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:49:31.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:31 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/2754469747' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:49:32.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:32 smithi006 ceph-mon[23740]: pgmap v43: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.4 MiB/s wr, 584 op/s 2024-09-17T10:49:32.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:32 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:32.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:32 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.plhvdm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:49:32.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:32 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.plhvdm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:49:32.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:32 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:49:32.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:32 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:49:32.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 98493 12.29 MB/sec execute 84 sec latency 36.303 ms 2024-09-17T10:49:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:32 smithi149 ceph-mon[26044]: pgmap v43: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.4 MiB/s wr, 584 op/s 2024-09-17T10:49:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:32 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:32 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.plhvdm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:49:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:32 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.plhvdm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:49:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:32 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:49:32.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:32 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:49:33.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:33 smithi006 ceph-mon[23740]: Upgrade: Updating mgr.smithi006.plhvdm 2024-09-17T10:49:33.683 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:33 smithi006 ceph-mon[23740]: Deploying daemon mgr.smithi006.plhvdm on smithi006 2024-09-17T10:49:33.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 98886 12.31 MB/sec execute 85 sec latency 49.212 ms 2024-09-17T10:49:33.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:33 smithi149 ceph-mon[26044]: Upgrade: Updating mgr.smithi006.plhvdm 2024-09-17T10:49:33.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:33 smithi149 ceph-mon[26044]: Deploying daemon mgr.smithi006.plhvdm on smithi006 2024-09-17T10:49:34.688 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:34 smithi006 ceph-mon[23740]: pgmap v44: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 836 op/s 2024-09-17T10:49:34.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 99295 12.36 MB/sec execute 86 sec latency 56.639 ms 2024-09-17T10:49:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:34 smithi149 ceph-mon[26044]: pgmap v44: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 836 op/s 2024-09-17T10:49:35.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 99660 12.34 MB/sec execute 87 sec latency 60.433 ms 2024-09-17T10:49:35.829 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:35 smithi006 ceph-mon[23740]: pgmap v45: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 571 op/s 2024-09-17T10:49:35.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:35 smithi149 ceph-mon[26044]: pgmap v45: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 571 op/s 2024-09-17T10:49:36.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 99940 12.27 MB/sec execute 88 sec latency 62.435 ms 2024-09-17T10:49:37.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 100284 12.21 MB/sec execute 89 sec latency 46.863 ms 2024-09-17T10:49:38.130 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:37 smithi006 ceph-mon[23740]: pgmap v46: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 571 op/s 2024-09-17T10:49:38.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:37 smithi149 ceph-mon[26044]: pgmap v46: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 571 op/s 2024-09-17T10:49:38.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 100551 12.11 MB/sec execute 90 sec latency 70.562 ms 2024-09-17T10:49:39.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 100873 12.08 MB/sec execute 91 sec latency 83.528 ms 2024-09-17T10:49:39.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:39 smithi006 ceph-mon[23740]: pgmap v47: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 790 op/s 2024-09-17T10:49:39.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:39 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:39.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:39 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:39.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:39 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:49:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:39 smithi149 ceph-mon[26044]: pgmap v47: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 790 op/s 2024-09-17T10:49:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:39 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:39 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:39 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:49:40.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 101246 12.01 MB/sec execute 92 sec latency 52.349 ms 2024-09-17T10:49:41.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:41 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:41.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:41 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:41.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:41 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:41.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:41 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:41.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 101733 12.02 MB/sec execute 93 sec latency 40.114 ms 2024-09-17T10:49:42.189 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:42 smithi006 ceph-mon[23740]: pgmap v48: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 470 op/s 2024-09-17T10:49:42.189 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:42 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:49:42.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:42 smithi149 ceph-mon[26044]: pgmap v48: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 470 op/s 2024-09-17T10:49:42.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:42 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:49:42.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 102147 12.06 MB/sec execute 94 sec latency 62.809 ms 2024-09-17T10:49:43.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 102495 12.06 MB/sec execute 95 sec latency 32.344 ms 2024-09-17T10:49:44.220 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:44 smithi006 ceph-mon[23740]: pgmap v49: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 691 op/s 2024-09-17T10:49:44.220 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:44 smithi006 ceph-mon[23740]: mgrmap e28: smithi149.ijckag(active, since 93s), standbys: smithi006.plhvdm 2024-09-17T10:49:44.220 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:44 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:44.220 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:44 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:44 smithi149 ceph-mon[26044]: pgmap v49: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 691 op/s 2024-09-17T10:49:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:44 smithi149 ceph-mon[26044]: mgrmap e28: smithi149.ijckag(active, since 93s), standbys: smithi006.plhvdm 2024-09-17T10:49:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:44 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:44.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:44 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:44.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 102852 12.07 MB/sec execute 96 sec latency 37.604 ms 2024-09-17T10:49:45.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 103138 12.03 MB/sec execute 97 sec latency 38.273 ms 2024-09-17T10:49:46.259 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:46 smithi006 ceph-mon[23740]: pgmap v50: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-09-17T10:49:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:46 smithi149 ceph-mon[26044]: pgmap v50: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-09-17T10:49:46.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 103355 11.98 MB/sec execute 98 sec latency 48.408 ms 2024-09-17T10:49:47.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 103585 11.89 MB/sec execute 99 sec latency 75.106 ms 2024-09-17T10:49:48.345 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:48 smithi006 ceph-mon[23740]: pgmap v51: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-09-17T10:49:48.345 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:48 smithi006 ceph-mon[23740]: Standby manager daemon smithi006.plhvdm restarted 2024-09-17T10:49:48.345 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:48 smithi006 ceph-mon[23740]: Standby manager daemon smithi006.plhvdm started 2024-09-17T10:49:48.345 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:48 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.plhvdm/crt"}]: dispatch 2024-09-17T10:49:48.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:48 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:49:48.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:48 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.plhvdm/key"}]: dispatch 2024-09-17T10:49:48.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:48 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:49:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:48 smithi149 ceph-mon[26044]: pgmap v51: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-09-17T10:49:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:48 smithi149 ceph-mon[26044]: Standby manager daemon smithi006.plhvdm restarted 2024-09-17T10:49:48.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:48 smithi149 ceph-mon[26044]: Standby manager daemon smithi006.plhvdm started 2024-09-17T10:49:48.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:48 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.plhvdm/crt"}]: dispatch 2024-09-17T10:49:48.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:48 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:49:48.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:48 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.plhvdm/key"}]: dispatch 2024-09-17T10:49:48.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:48 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:49:48.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 103863 11.82 MB/sec execute 100 sec latency 103.788 ms 2024-09-17T10:49:49.353 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:49 smithi006 ceph-mon[23740]: mgrmap e29: smithi149.ijckag(active, since 98s), standbys: smithi006.plhvdm 2024-09-17T10:49:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:49 smithi149 ceph-mon[26044]: mgrmap e29: smithi149.ijckag(active, since 98s), standbys: smithi006.plhvdm 2024-09-17T10:49:49.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 104177 11.74 MB/sec execute 101 sec latency 44.304 ms 2024-09-17T10:49:50.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:50 smithi006 ceph-mon[23740]: pgmap v52: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.9 MiB/s wr, 634 op/s 2024-09-17T10:49:50.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:50 smithi149 ceph-mon[26044]: pgmap v52: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.9 MiB/s wr, 634 op/s 2024-09-17T10:49:50.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 104766 11.77 MB/sec execute 102 sec latency 38.257 ms 2024-09-17T10:49:51.382 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:51 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:51.382 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:51 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:51.382 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:51 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:51.382 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:51 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:51.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:51 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:51.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:51 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:51.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:51 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:51.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:51 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:51.796 INFO:teuthology.orchestra.run.smithi006.stdout: 5 105336 11.80 MB/sec execute 103 sec latency 22.795 ms 2024-09-17T10:49:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:52 smithi006 ceph-mon[23740]: pgmap v53: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-09-17T10:49:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:52 smithi149 ceph-mon[26044]: pgmap v53: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-09-17T10:49:52.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 105813 11.86 MB/sec execute 104 sec latency 35.971 ms 2024-09-17T10:49:53.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 106228 11.90 MB/sec execute 105 sec latency 38.743 ms 2024-09-17T10:49:54.426 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:54 smithi006 ceph-mon[23740]: pgmap v54: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 653 op/s 2024-09-17T10:49:54.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:54 smithi149 ceph-mon[26044]: pgmap v54: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 653 op/s 2024-09-17T10:49:54.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 106724 11.93 MB/sec execute 106 sec latency 43.500 ms 2024-09-17T10:49:55.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 107160 11.90 MB/sec execute 107 sec latency 31.260 ms 2024-09-17T10:49:56.444 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:56 smithi006 ceph-mon[23740]: pgmap v55: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 432 op/s 2024-09-17T10:49:56.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:56 smithi149 ceph-mon[26044]: pgmap v55: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 432 op/s 2024-09-17T10:49:56.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 107620 11.86 MB/sec execute 108 sec latency 35.727 ms 2024-09-17T10:49:57.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 107968 11.84 MB/sec execute 109 sec latency 51.104 ms 2024-09-17T10:49:57.982 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:57 smithi006 ceph-mon[23740]: pgmap v56: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 432 op/s 2024-09-17T10:49:57.983 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:57 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:49:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:57 smithi149 ceph-mon[26044]: pgmap v56: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 432 op/s 2024-09-17T10:49:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:57 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:49:58.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 108306 11.79 MB/sec execute 110 sec latency 39.625 ms 2024-09-17T10:49:59.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: Upgrade: Need to upgrade myself (mgr.smithi149.ijckag) 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: Upgrade: Need to upgrade myself (mgr.smithi149.ijckag) 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: Failing over to other MGR 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr fail", "who": "smithi149.ijckag"}]: dispatch 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr fail", "who": "smithi149.ijckag"}]: dispatch 2024-09-17T10:49:59.003 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:49:58 smithi006 ceph-mon[23740]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T10:49:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:49:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: Upgrade: Need to upgrade myself (mgr.smithi149.ijckag) 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: Upgrade: Need to upgrade myself (mgr.smithi149.ijckag) 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: Failing over to other MGR 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 172.21.15.149:0/3877499604' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr fail", "who": "smithi149.ijckag"}]: dispatch 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "mgr fail", "who": "smithi149.ijckag"}]: dispatch 2024-09-17T10:49:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:49:58 smithi149 ceph-mon[26044]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T10:49:59.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 108652 11.77 MB/sec execute 111 sec latency 65.660 ms 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd='[{"prefix": "mgr fail", "who": "smithi149.ijckag"}]': finished 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: mgrmap e30: smithi006.plhvdm(active, starting, since 0.610534s) 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"}]: dispatch 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"}]: dispatch 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:50:00.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: Manager daemon smithi006.plhvdm is now available 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:50:00.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:50:00.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:50:00.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:00 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.24445 ' entity='mgr.smithi149.ijckag' cmd='[{"prefix": "mgr fail", "who": "smithi149.ijckag"}]': finished 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: mgrmap e30: smithi006.plhvdm(active, starting, since 0.610534s) 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"}]: dispatch 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"}]: dispatch 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"}]: dispatch 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T10:50:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: Manager daemon smithi006.plhvdm is now available 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T10:50:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:00 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"}]: dispatch 2024-09-17T10:50:00.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 108949 11.70 MB/sec execute 112 sec latency 67.036 ms 2024-09-17T10:50:01.065 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:01 smithi006 ceph-mon[23740]: overall HEALTH_OK 2024-09-17T10:50:01.065 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:01 smithi006 ceph-mon[23740]: mgrmap e31: smithi006.plhvdm(active, since 1.6596s) 2024-09-17T10:50:01.065 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:01 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:50:01.065 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:01 smithi006 ceph-mon[23740]: pgmap v3: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:50:01.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:01 smithi149 ceph-mon[26044]: overall HEALTH_OK 2024-09-17T10:50:01.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:01 smithi149 ceph-mon[26044]: mgrmap e31: smithi006.plhvdm(active, since 1.6596s) 2024-09-17T10:50:01.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:01 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:50:01.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:01 smithi149 ceph-mon[26044]: pgmap v3: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:50:01.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 109327 11.74 MB/sec execute 113 sec latency 46.187 ms 2024-09-17T10:50:01.951 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:50:02.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:02 smithi006 ceph-mon[23740]: [17/Sep/2024:10:50:01] ENGINE Bus STARTING 2024-09-17T10:50:02.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:02 smithi006 ceph-mon[23740]: [17/Sep/2024:10:50:01] ENGINE Serving on http://172.21.15.6:8765 2024-09-17T10:50:02.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:02 smithi006 ceph-mon[23740]: [17/Sep/2024:10:50:01] ENGINE Serving on https://172.21.15.6:7150 2024-09-17T10:50:02.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:02 smithi006 ceph-mon[23740]: [17/Sep/2024:10:50:01] ENGINE Bus STARTED 2024-09-17T10:50:02.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:02 smithi006 ceph-mon[23740]: [17/Sep/2024:10:50:01] ENGINE Error in HTTPServer.serve 2024-09-17T10:50:02.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Traceback (most recent call last): 2024-09-17T10:50:02.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T10:50:02.084 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._connections.run(self.expiration_interval) 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._run(expiration_interval) 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: s = self.context.wrap_socket( 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: return self.sslsocket_class._create( 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self.do_handshake() 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T10:50:02.085 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._sslobj.do_handshake() 2024-09-17T10:50:02.086 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:50:01] ENGINE Bus STARTING 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:50:01] ENGINE Serving on http://172.21.15.6:8765 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:50:01] ENGINE Serving on https://172.21.15.6:7150 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:50:01] ENGINE Bus STARTED 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:50:01] ENGINE Error in HTTPServer.serve 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Traceback (most recent call last): 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._connections.run(self.expiration_interval) 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T10:50:02.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._run(expiration_interval) 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: s = self.context.wrap_socket( 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: return self.sslsocket_class._create( 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self.do_handshake() 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._sslobj.do_handshake() 2024-09-17T10:50:02.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T10:50:02.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 109653 11.73 MB/sec execute 114 sec latency 42.490 ms 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (81s) 12s ago 10m 16.2M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (10m) 12s ago 10m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (9m) 19s ago 9m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (70s) 12s ago 10m 46.0M - dad864ee21e9 89df198e1a9b 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (4m) 12s ago 4m 36.1M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (4m) 19s ago 4m 13.1M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (24s) 12s ago 11m 88.1M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:50:02.925 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283 running (2m) 19s ago 9m 517M - 19.3.0-4937-g74984932 6c1fcab85602 053d414e8a22 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (11m) 12s ago 11m 55.0M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (9m) 19s ago 9m 41.3M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (4m) 12s ago 4m 264M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (4m) 19s ago 4m 81.3M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (62s) 12s ago 10m 9.76M - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (34s) 19s ago 9m 7314k - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (8m) 12s ago 8m 406M 1937M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (8m) 12s ago 8m 302M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (7m) 12s ago 7m 266M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (7m) 12s ago 7m 370M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (7m) 19s ago 7m 353M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (6m) 19s ago 6m 397M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:50:02.926 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (6m) 19s ago 6m 338M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:50:02.927 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (6m) 19s ago 6m 464M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:50:02.927 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (40s) 12s ago 9m 39.7M - 2.51.0 1d3b7f56885b 6ba03bbc6f1c 2024-09-17T10:50:03.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:03 smithi006 ceph-mon[23740]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:03.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:03 smithi006 ceph-mon[23740]: mgrmap e32: smithi006.plhvdm(active, since 3s) 2024-09-17T10:50:03.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:03 smithi006 ceph-mon[23740]: from='client.14810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:03 smithi149 ceph-mon[26044]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:03 smithi149 ceph-mon[26044]: mgrmap e32: smithi006.plhvdm(active, since 3s) 2024-09-17T10:50:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:03 smithi149 ceph-mon[26044]: from='client.14810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:03.490 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:50:03.490 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:50:03.490 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:50:03.490 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:50:03.490 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:50:03.491 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:50:03.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 109938 11.74 MB/sec execute 115 sec latency 51.551 ms 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T10:50:03.977 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-09-17T10:50:03.978 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:50:03.978 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:50:04.142 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:04 smithi006 ceph-mon[23740]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:04.143 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:04 smithi006 ceph-mon[23740]: pgmap v5: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:50:04.143 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:04 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3938772388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:50:04.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:04 smithi149 ceph-mon[26044]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:04.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:04 smithi149 ceph-mon[26044]: pgmap v5: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:50:04.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:04 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3938772388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:50:04.605 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:50:04.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 110240 11.72 MB/sec execute 116 sec latency 56.844 ms 2024-09-17T10:50:05.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:05 smithi006 ceph-mon[23740]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:05.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:05 smithi006 ceph-mon[23740]: mgrmap e33: smithi006.plhvdm(active, since 5s) 2024-09-17T10:50:05.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:05 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/3374538979' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:50:05.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:05 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:05.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:05 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:05 smithi149 ceph-mon[26044]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:05 smithi149 ceph-mon[26044]: mgrmap e33: smithi006.plhvdm(active, since 5s) 2024-09-17T10:50:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:05 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/3374538979' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:50:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:05 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:05 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:05.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 110526 11.67 MB/sec execute 117 sec latency 50.240 ms 2024-09-17T10:50:06.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: pgmap v6: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:50:06.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:06.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:06.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:50:06.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:50:06.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/1478439460' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:50:06.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/1478439460' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:50:06.415 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/1478439460' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:50:06.415 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:06 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/1478439460' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:50:06.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: pgmap v6: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-17T10:50:06.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:06.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:06.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-09-17T10:50:06.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:50:06.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/1478439460' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:50:06.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/1478439460' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:50:06.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/1478439460' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:50:06.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:06 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/1478439460' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:50:06.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 110773 11.60 MB/sec execute 118 sec latency 63.982 ms 2024-09-17T10:50:07.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:07 smithi006 ceph-mon[23740]: mgrmap e34: smithi006.plhvdm(active, since 8s), standbys: smithi149.ijckag 2024-09-17T10:50:07.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:07 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi149.ijckag", "id": "smithi149.ijckag"}]: dispatch 2024-09-17T10:50:07.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:07 smithi149 ceph-mon[26044]: mgrmap e34: smithi006.plhvdm(active, since 8s), standbys: smithi149.ijckag 2024-09-17T10:50:07.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:07 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr metadata", "who": "smithi149.ijckag", "id": "smithi149.ijckag"}]: dispatch 2024-09-17T10:50:07.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 111136 11.56 MB/sec execute 119 sec latency 49.322 ms 2024-09-17T10:50:08.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:08 smithi006 ceph-mon[23740]: pgmap v7: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 MiB/s wr, 317 op/s 2024-09-17T10:50:08.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:08 smithi149 ceph-mon[26044]: pgmap v7: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 MiB/s wr, 317 op/s 2024-09-17T10:50:08.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 111501 11.54 MB/sec execute 120 sec latency 41.950 ms 2024-09-17T10:50:09.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 111909 11.51 MB/sec execute 121 sec latency 33.953 ms 2024-09-17T10:50:10.486 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:10 smithi006 ceph-mon[23740]: pgmap v8: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 3.1 MiB/s wr, 246 op/s 2024-09-17T10:50:10.487 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:10 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:10.487 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:10 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:10.797 INFO:teuthology.orchestra.run.smithi006.stdout: 5 112357 11.51 MB/sec execute 122 sec latency 44.474 ms 2024-09-17T10:50:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:10 smithi149 ceph-mon[26044]: pgmap v8: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 3.1 MiB/s wr, 246 op/s 2024-09-17T10:50:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:10 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:10 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:11.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 113002 11.58 MB/sec execute 123 sec latency 26.400 ms 2024-09-17T10:50:12.019 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:11 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:12.019 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:11 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:12.019 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:11 smithi006 ceph-mon[23740]: pgmap v9: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-17T10:50:12.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:11 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:12.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:11 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:12.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:11 smithi149 ceph-mon[26044]: pgmap v9: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-17T10:50:12.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 113325 11.62 MB/sec execute 124 sec latency 48.911 ms 2024-09-17T10:50:13.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 113772 11.64 MB/sec execute 125 sec latency 38.157 ms 2024-09-17T10:50:14.296 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:14 smithi006 ceph-mon[23740]: pgmap v10: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T10:50:14.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:14 smithi149 ceph-mon[26044]: pgmap v10: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T10:50:14.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 114096 11.60 MB/sec execute 126 sec latency 37.971 ms 2024-09-17T10:50:15.074 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:15 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:50:15.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:15 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:50:15.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 114425 11.55 MB/sec execute 127 sec latency 41.650 ms 2024-09-17T10:50:16.090 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:16 smithi006 ceph-mon[23740]: pgmap v11: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T10:50:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:16 smithi149 ceph-mon[26044]: pgmap v11: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T10:50:16.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 114851 11.55 MB/sec execute 128 sec latency 73.748 ms 2024-09-17T10:50:17.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 115303 11.52 MB/sec execute 129 sec latency 45.126 ms 2024-09-17T10:50:18.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 7.0 MiB/s wr, 650 op/s 2024-09-17T10:50:18.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:50:18.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:50:18.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:50:18.118 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: Updating smithi149:/etc/ceph/ceph.conf 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.119 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:18 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 7.0 MiB/s wr, 650 op/s 2024-09-17T10:50:18.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: Updating smithi149:/etc/ceph/ceph.conf 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:18 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:18.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 115696 11.49 MB/sec execute 130 sec latency 45.168 ms 2024-09-17T10:50:19.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:19 smithi006 ceph-mon[23740]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:50:19.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:19 smithi006 ceph-mon[23740]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:50:19.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:19 smithi006 ceph-mon[23740]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:50:19.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:19 smithi006 ceph-mon[23740]: Reconfiguring prometheus.smithi006 (dependencies changed)... 2024-09-17T10:50:19.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:19 smithi006 ceph-mon[23740]: Reconfiguring daemon prometheus.smithi006 on smithi006 2024-09-17T10:50:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:19 smithi149 ceph-mon[26044]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:50:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:19 smithi149 ceph-mon[26044]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:50:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:19 smithi149 ceph-mon[26044]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:50:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:19 smithi149 ceph-mon[26044]: Reconfiguring prometheus.smithi006 (dependencies changed)... 2024-09-17T10:50:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:19 smithi149 ceph-mon[26044]: Reconfiguring daemon prometheus.smithi006 on smithi006 2024-09-17T10:50:19.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 116253 11.53 MB/sec execute 131 sec latency 49.593 ms 2024-09-17T10:50:20.161 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:20 smithi006 ceph-mon[23740]: pgmap v13: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 468 op/s 2024-09-17T10:50:20.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:20 smithi149 ceph-mon[26044]: pgmap v13: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 468 op/s 2024-09-17T10:50:20.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 116738 11.58 MB/sec execute 132 sec latency 31.442 ms 2024-09-17T10:50:21.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 117469 11.69 MB/sec execute 133 sec latency 32.646 ms 2024-09-17T10:50:22.178 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:22 smithi006 ceph-mon[23740]: pgmap v14: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 736 op/s 2024-09-17T10:50:22.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:22 smithi149 ceph-mon[26044]: pgmap v14: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 736 op/s 2024-09-17T10:50:22.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 118142 11.69 MB/sec execute 134 sec latency 25.244 ms 2024-09-17T10:50:23.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 118939 11.72 MB/sec execute 135 sec latency 20.702 ms 2024-09-17T10:50:24.190 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:24 smithi006 ceph-mon[23740]: pgmap v15: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 516 op/s 2024-09-17T10:50:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:24 smithi149 ceph-mon[26044]: pgmap v15: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 516 op/s 2024-09-17T10:50:24.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 119421 11.72 MB/sec execute 136 sec latency 47.163 ms 2024-09-17T10:50:25.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 119855 11.74 MB/sec execute 137 sec latency 35.640 ms 2024-09-17T10:50:26.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:26 smithi006 ceph-mon[23740]: pgmap v16: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 516 op/s 2024-09-17T10:50:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:26 smithi149 ceph-mon[26044]: pgmap v16: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 516 op/s 2024-09-17T10:50:26.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 120232 11.76 MB/sec execute 138 sec latency 82.470 ms 2024-09-17T10:50:27.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 120781 11.84 MB/sec execute 139 sec latency 33.409 ms 2024-09-17T10:50:28.294 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:28 smithi006 ceph-mon[23740]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 884 op/s 2024-09-17T10:50:28.294 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:28 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:28.294 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:28 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:28.294 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:28 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:50:28.294 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:28 smithi006 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:50:28.294 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:28 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:50:28.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:28 smithi149 ceph-mon[26044]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 884 op/s 2024-09-17T10:50:28.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:28 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:28.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:28 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:28.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:28 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:50:28.445 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:28 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T10:50:28.445 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:28 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:50:28.799 INFO:teuthology.orchestra.run.smithi006.stdout: 5 121034 11.80 MB/sec execute 140 sec latency 80.743 ms 2024-09-17T10:50:29.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 121299 11.73 MB/sec execute 141 sec latency 77.163 ms 2024-09-17T10:50:30.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:30 smithi006 ceph-mon[23740]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.2 MiB/s wr, 635 op/s 2024-09-17T10:50:30.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:30 smithi006 ceph-mon[23740]: Upgrade: Updating mgr.smithi149.ijckag 2024-09-17T10:50:30.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:30 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:50:30.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:30 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:50:30.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:30 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:50:30.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:30 smithi006 ceph-mon[23740]: Deploying daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:50:30.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:30 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:50:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:30 smithi149 ceph-mon[26044]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.2 MiB/s wr, 635 op/s 2024-09-17T10:50:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:30 smithi149 ceph-mon[26044]: Upgrade: Updating mgr.smithi149.ijckag 2024-09-17T10:50:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:30 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T10:50:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:30 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T10:50:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:30 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:50:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:30 smithi149 ceph-mon[26044]: Deploying daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:50:30.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:30 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:50:30.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 121611 11.70 MB/sec execute 142 sec latency 82.790 ms 2024-09-17T10:50:31.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 121976 11.68 MB/sec execute 143 sec latency 44.682 ms 2024-09-17T10:50:32.215 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:32 smithi006 ceph-mon[23740]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 861 op/s 2024-09-17T10:50:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:32 smithi149 ceph-mon[26044]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 861 op/s 2024-09-17T10:50:32.798 INFO:teuthology.orchestra.run.smithi006.stdout: 5 122523 11.67 MB/sec execute 144 sec latency 61.684 ms 2024-09-17T10:50:33.799 INFO:teuthology.orchestra.run.smithi006.stdout: 5 123377 11.74 MB/sec execute 145 sec latency 37.502 ms 2024-09-17T10:50:34.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:34 smithi006 ceph-mon[23740]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.4 MiB/s wr, 593 op/s 2024-09-17T10:50:34.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:34 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:34.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:34 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:34.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:34 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:50:34.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:34 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:34.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:34 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:34 smithi149 ceph-mon[26044]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.4 MiB/s wr, 593 op/s 2024-09-17T10:50:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:34 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:34 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:34 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:50:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:34 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:34 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:34.799 INFO:teuthology.orchestra.run.smithi006.stdout: 5 123861 11.80 MB/sec execute 146 sec latency 38.109 ms 2024-09-17T10:50:35.086 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:50:35.799 INFO:teuthology.orchestra.run.smithi006.stdout: 5 124160 11.80 MB/sec execute 147 sec latency 45.956 ms 2024-09-17T10:50:36.031 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (114s) 26s ago 11m 16.3M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (10m) 26s ago 10m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (10m) 31s ago 10m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (104s) 26s ago 10m 44.8M - dad864ee21e9 89df198e1a9b 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (5m) 26s ago 5m 37.1M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (4m) 31s ago 4m 13.5M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (57s) 26s ago 12m 489M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 starting - - - - 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (12m) 26s ago 12m 59.4M 2048M 17.2.0 e1d6a67b021e 12038485a6a7 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (10m) 31s ago 10m 43.6M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (4m) 26s ago 4m 261M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (4m) 31s ago 4m 83.7M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:50:36.032 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (96s) 26s ago 10m 9445k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (67s) 31s ago 10m 7692k - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (9m) 26s ago 9m 426M 1937M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (8m) 26s ago 8m 325M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (8m) 26s ago 8m 293M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (8m) 26s ago 8m 396M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (7m) 31s ago 7m 381M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (7m) 31s ago 7m 422M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (6m) 31s ago 6m 359M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (6m) 31s ago 6m 484M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:50:36.033 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 starting - - - - 2024-09-17T10:50:36.271 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:36 smithi006 ceph-mon[23740]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.4 MiB/s wr, 593 op/s 2024-09-17T10:50:36.272 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:36 smithi006 ceph-mon[23740]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:36.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:36 smithi149 ceph-mon[26044]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.4 MiB/s wr, 593 op/s 2024-09-17T10:50:36.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:36 smithi149 ceph-mon[26044]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:36.585 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:50:36.585 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:50:36.585 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:50:36.585 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:50:36.585 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:50:36.585 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:50:36.586 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:50:36.806 INFO:teuthology.orchestra.run.smithi006.stdout: 5 124415 11.77 MB/sec execute 148 sec latency 87.631 ms 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "", 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mgr daemons", 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:50:37.035 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:50:37.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:37 smithi149 ceph-mon[26044]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:37.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:37 smithi149 ceph-mon[26044]: from='client.14842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:37.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:37 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1981236373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:50:37.279 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:37 smithi006 ceph-mon[23740]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:37.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:37 smithi006 ceph-mon[23740]: from='client.14842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:37.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:37 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/1981236373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:50:37.562 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:50:37.806 INFO:teuthology.orchestra.run.smithi006.stdout: 5 124616 11.73 MB/sec execute 149 sec latency 55.404 ms 2024-09-17T10:50:38.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:38 smithi149 ceph-mon[26044]: pgmap v22: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 892 op/s 2024-09-17T10:50:38.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:38 smithi149 ceph-mon[26044]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:38.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:38 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/50856597' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:50:38.304 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:38 smithi006 ceph-mon[23740]: pgmap v22: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 892 op/s 2024-09-17T10:50:38.305 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:38 smithi006 ceph-mon[23740]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:50:38.305 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:38 smithi006 ceph-mon[23740]: from='client.? 172.21.15.6:0/50856597' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T10:50:38.806 INFO:teuthology.orchestra.run.smithi006.stdout: 5 124872 11.67 MB/sec execute 150 sec latency 53.846 ms 2024-09-17T10:50:39.806 INFO:teuthology.orchestra.run.smithi006.stdout: 5 125173 11.63 MB/sec execute 151 sec latency 59.726 ms 2024-09-17T10:50:39.838 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:39 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:39.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:39 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:39.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:39 smithi006 ceph-mon[23740]: pgmap v23: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 525 op/s 2024-09-17T10:50:39.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:39 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:39.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:39 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:39 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:39 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:39 smithi149 ceph-mon[26044]: pgmap v23: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 525 op/s 2024-09-17T10:50:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:39 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:39 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:40.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 125466 11.59 MB/sec execute 152 sec latency 67.318 ms 2024-09-17T10:50:40.852 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:40 smithi006 ceph-mon[23740]: Standby manager daemon smithi149.ijckag restarted 2024-09-17T10:50:40.852 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:40 smithi006 ceph-mon[23740]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:50:40.852 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:40 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/4209036972' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:50:40.852 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:40 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/4209036972' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:50:40.852 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:40 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/4209036972' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:50:40.852 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:40 smithi006 ceph-mon[23740]: from='mgr.? 172.21.15.149:0/4209036972' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:50:40.954 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:40 smithi149 ceph-mon[26044]: Standby manager daemon smithi149.ijckag restarted 2024-09-17T10:50:40.954 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:40 smithi149 ceph-mon[26044]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:50:40.954 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:40 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/4209036972' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:50:40.954 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:40 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/4209036972' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:50:40.954 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:40 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/4209036972' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:50:40.954 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:40 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/4209036972' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:50:41.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 125807 11.57 MB/sec execute 153 sec latency 53.924 ms 2024-09-17T10:50:41.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:41 smithi006 ceph-mon[23740]: mgrmap e35: smithi006.plhvdm(active, since 42s), standbys: smithi149.ijckag 2024-09-17T10:50:41.866 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:41 smithi006 ceph-mon[23740]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 693 op/s 2024-09-17T10:50:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:41 smithi149 ceph-mon[26044]: mgrmap e35: smithi006.plhvdm(active, since 42s), standbys: smithi149.ijckag 2024-09-17T10:50:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:41 smithi149 ceph-mon[26044]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 693 op/s 2024-09-17T10:50:42.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 126201 11.55 MB/sec execute 154 sec latency 67.187 ms 2024-09-17T10:50:43.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 126557 11.53 MB/sec execute 155 sec latency 64.736 ms 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: pgmap v25: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.449 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:50:44.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:50:44.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi006.plhvdm"}]: dispatch 2024-09-17T10:50:44.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi006.plhvdm"}]': finished 2024-09-17T10:50:44.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi149.ijckag"}]: dispatch 2024-09-17T10:50:44.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi149.ijckag"}]': finished 2024-09-17T10:50:44.450 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:44 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T10:50:44.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: pgmap v25: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-09-17T10:50:44.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi006.plhvdm"}]: dispatch 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi006.plhvdm"}]': finished 2024-09-17T10:50:44.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi149.ijckag"}]: dispatch 2024-09-17T10:50:44.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi149.ijckag"}]': finished 2024-09-17T10:50:44.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:44 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T10:50:44.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 126979 11.54 MB/sec execute 156 sec latency 37.763 ms 2024-09-17T10:50:45.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:45 smithi006 ceph-mon[23740]: Upgrade: Setting container_image for all mgr 2024-09-17T10:50:45.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:45 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:50:45.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:45 smithi149 ceph-mon[26044]: Upgrade: Setting container_image for all mgr 2024-09-17T10:50:45.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:45 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T10:50:45.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 127528 11.63 MB/sec execute 157 sec latency 26.167 ms 2024-09-17T10:50:46.474 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:46 smithi006 ceph-mon[23740]: pgmap v26: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-09-17T10:50:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:46 smithi149 ceph-mon[26044]: pgmap v26: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-09-17T10:50:46.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 127940 11.63 MB/sec execute 158 sec latency 44.986 ms 2024-09-17T10:50:47.740 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:47 smithi006 ceph-mon[23740]: Upgrade: Updating mon.smithi006 2024-09-17T10:50:47.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:47 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:47.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:47 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T10:50:47.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:47 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T10:50:47.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:47 smithi006 ceph-mon[23740]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:50:47.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:47 smithi006 ceph-mon[23740]: Deploying daemon mon.smithi006 on smithi006 2024-09-17T10:50:47.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:47 smithi006 ceph-mon[23740]: pgmap v27: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 682 op/s 2024-09-17T10:50:47.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 128226 11.60 MB/sec execute 159 sec latency 53.151 ms 2024-09-17T10:50:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:47 smithi149 ceph-mon[26044]: Upgrade: Updating mon.smithi006 2024-09-17T10:50:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:47 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' 2024-09-17T10:50:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:47 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T10:50:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:47 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T10:50:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:47 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T10:50:47.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:47 smithi149 ceph-mon[26044]: Deploying daemon mon.smithi006 on smithi006 2024-09-17T10:50:47.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:47 smithi149 ceph-mon[26044]: pgmap v27: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 682 op/s 2024-09-17T10:50:48.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 128459 11.54 MB/sec execute 160 sec latency 48.299 ms 2024-09-17T10:50:49.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:49 smithi006 systemd[1]: Stopping Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:50:49.794 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:49 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[23736]: 2024-09-17T10:50:49.676+0000 7f3dd9a49700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi006 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:50:49.794 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:49 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[23736]: 2024-09-17T10:50:49.676+0000 7f3dd9a49700 -1 mon.smithi006@0(leader) e2 *** Got Signal Terminated *** 2024-09-17T10:50:49.794 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:49 smithi006 podman[63801]: 2024-09-17 10:50:49.698684574 +0000 UTC m=+0.158619481 container died 12038485a6a786222af5d3e206d29ae96d04b4fe184d1019539457e541beae3b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, RELEASE=HEAD, io.buildah.version=1.19.8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, distribution-scope=public, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T10:50:49.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 128738 11.51 MB/sec execute 161 sec latency 61.716 ms 2024-09-17T10:50:50.304 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:50 smithi006 podman[63801]: 2024-09-17 10:50:50.288204424 +0000 UTC m=+0.748139319 container cleanup 12038485a6a786222af5d3e206d29ae96d04b4fe184d1019539457e541beae3b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, version=8, GIT_CLEAN=True, name=centos-stream, distribution-scope=public, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, release=754, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-17T10:50:50.304 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:50 smithi006 bash[63801]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006 2024-09-17T10:50:50.806 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:50 smithi006 podman[63813]: 2024-09-17 10:50:50.616683589 +0000 UTC m=+0.910853388 container remove 12038485a6a786222af5d3e206d29ae96d04b4fe184d1019539457e541beae3b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD) 2024-09-17T10:50:50.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 129029 11.47 MB/sec execute 162 sec latency 56.645 ms 2024-09-17T10:50:51.067 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:50 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006.service: Deactivated successfully. 2024-09-17T10:50:51.067 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:50 smithi006 systemd[1]: Stopped Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:50:51.067 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:50 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006.service: Consumed 17.713s CPU time. 2024-09-17T10:50:51.339 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:51 smithi006 systemd[1]: Starting Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:50:51.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:51 smithi006 podman[63946]: 2024-09-17 10:50:51.451788808 +0000 UTC m=+0.018711986 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:50:51.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 129319 11.44 MB/sec execute 163 sec latency 92.376 ms 2024-09-17T10:50:51.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:51 smithi006 podman[63946]: 2024-09-17 10:50:51.702021881 +0000 UTC m=+0.268945058 container create 241e5f24b4c7e5602cf542a79e735ae6905975938c64bf0e77276a331f169c1d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.license=GPLv2) 2024-09-17T10:50:52.125 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 podman[63946]: 2024-09-17 10:50:52.114456123 +0000 UTC m=+0.681379313 container init 241e5f24b4c7e5602cf542a79e735ae6905975938c64bf0e77276a331f169c1d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-7498493) 2024-09-17T10:50:52.125 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 podman[63946]: 2024-09-17 10:50:52.123839442 +0000 UTC m=+0.690762625 container start 241e5f24b4c7e5602cf542a79e735ae6905975938c64bf0e77276a331f169c1d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T10:50:52.382 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T10:50:52.382 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev), process ceph-mon, pid 2 2024-09-17T10:50:52.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: pidfile_write: ignore empty --pid-file 2024-09-17T10:50:52.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: load: jerasure load: lrc 2024-09-17T10:50:52.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: RocksDB version: 7.9.2 2024-09-17T10:50:52.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Git sha 0 2024-09-17T10:50:52.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Compile date 2024-09-15 20:52:36 2024-09-17T10:50:52.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: DB SUMMARY 2024-09-17T10:50:52.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: DB Session ID: 0TUTD7R9XL9AQ1O649CI 2024-09-17T10:50:52.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: CURRENT file: CURRENT 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi006/store.db dir, Total Num: 1, files: 000030.sst 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi006/store.db: 000028.log size: 2432700 ; 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.error_if_exists: 0 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.create_if_missing: 0 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.paranoid_checks: 1 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T10:50:52.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T10:50:52.385 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T10:50:52.385 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.env: 0x55bcfd07f120 2024-09-17T10:50:52.385 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T10:50:52.385 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.info_log: 0x55bcfeed18e0 2024-09-17T10:50:52.385 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T10:50:52.385 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.statistics: (nil) 2024-09-17T10:50:52.385 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.use_fsync: 0 2024-09-17T10:50:52.385 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_log_file_size: 0 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.allow_fallocate: 1 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.use_direct_reads: 0 2024-09-17T10:50:52.386 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T10:50:52.387 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T10:50:52.387 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.db_log_dir: 2024-09-17T10:50:52.387 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.wal_dir: 2024-09-17T10:50:52.387 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T10:50:52.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T10:50:52.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T10:50:52.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T10:50:52.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T10:50:52.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T10:50:52.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.write_buffer_manager: 0x55bcfeed5b80 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T10:50:52.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.unordered_write: 0 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.row_cache: None 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.wal_filter: None 2024-09-17T10:50:52.390 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T10:50:52.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T10:50:52.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.two_write_queues: 0 2024-09-17T10:50:52.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T10:50:52.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.wal_compression: 0 2024-09-17T10:50:52.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.atomic_flush: 0 2024-09-17T10:50:52.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T10:50:52.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T10:50:52.391 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.log_readahead_size: 0 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_background_jobs: 2 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_background_compactions: -1 2024-09-17T10:50:52.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_subcompactions: 1 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_open_files: -1 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T10:50:52.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_background_flushes: -1 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Compression algorithms supported: 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: kZSTD supported: 0 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: kXpressCompression supported: 0 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: kBZip2Compression supported: 0 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: kLZ4Compression supported: 1 2024-09-17T10:50:52.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: kZlibCompression supported: 1 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: kSnappyCompression supported: 1 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi006/store.db/MANIFEST-000015 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.merge_operator: 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_filter: None 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T10:50:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T10:50:52.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bcfeed1540) 2024-09-17T10:50:52.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: cache_index_and_filter_blocks: 1 2024-09-17T10:50:52.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T10:50:52.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T10:50:52.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: pin_top_level_index_and_filter: 1 2024-09-17T10:50:52.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_type: 0 2024-09-17T10:50:52.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_block_index_type: 0 2024-09-17T10:50:52.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_shortening: 1 2024-09-17T10:50:52.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T10:50:52.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: checksum: 4 2024-09-17T10:50:52.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: no_block_cache: 0 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache: 0x55bcfeec5610 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_name: BinnedLRUCache 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_options: 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: capacity : 536870912 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: num_shard_bits : 4 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: strict_capacity_limit : 0 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: high_pri_pool_ratio: 0.000 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_compressed: (nil) 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: persistent_cache: (nil) 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_size: 4096 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_size_deviation: 10 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_restart_interval: 16 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_block_restart_interval: 1 2024-09-17T10:50:52.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: metadata_block_size: 4096 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: partition_filters: 0 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: use_delta_encoding: 1 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: filter_policy: bloomfilter 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: whole_key_filtering: 1 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: verify_compression: 0 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: read_amp_bytes_per_bit: 0 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: format_version: 5 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: enable_index_compression: 1 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_align: 0 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: max_auto_readahead_size: 262144 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: prepopulate_block_cache: 0 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: initial_auto_readahead_size: 8192 2024-09-17T10:50:52.399 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression: NoCompression 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.num_levels: 7 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T10:50:52.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T10:50:52.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T10:50:52.402 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T10:50:52.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.inplace_update_support: 0 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T10:50:52.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.bloom_locality: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.max_successive_merges: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.ttl: 2592000 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.enable_blob_files: false 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.min_blob_size: 0 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T10:50:52.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi006/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11853, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9a09a1d8-ace3-494c-a9e4-8ad9b832c557 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726570252155092, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-17T10:50:52.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726570252164757, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2233746, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11854, "largest_seqno": 12888, "table_properties": {"data_size": 2228399, "index_size": 3150, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10436, "raw_average_key_size": 23, "raw_value_size": 2218534, "raw_average_value_size": 4930, "num_data_blocks": 144, "num_entries": 450, "num_filter_entries": 450, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726570252, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9a09a1d8-ace3-494c-a9e4-8ad9b832c557", "db_session_id": "0TUTD7R9XL9AQ1O649CI", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726570252164953, "job": 1, "event": "recovery_finished"} 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi006/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55bcfeef4e00 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: DB pointer 0x55bcfeff6000 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** DB Stats ** 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T10:50:52.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: L0 1/0 2.13 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 465.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: L6 1/0 6.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Sum 2/0 8.91 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 465.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-17T10:50:52.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 465.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 465.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T10:50:52.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative compaction: 0.00 GB write, 147.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval compaction: 0.00 GB write, 147.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Block cache BinnedLRUCache@0x55bcfeec5610#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: starting mon.smithi006 rank 0 at public addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] at bind addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi006 fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006@-1(???) e2 preinit fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:50:52.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006@-1(???).mds e7 new map 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006@-1(???).mds e7 print_map 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: e7 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: legacy client fscid: 1 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Filesystem 'foofs' (1) 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: fs_name foofs 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: epoch 5 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: created 2024-09-17T10:45:30.960183+0000 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: modified 2024-09-17T10:45:36.993299+0000 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: tableserver 0 2024-09-17T10:50:52.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: root 0 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: session_timeout 60 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: session_autoclose 300 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: max_file_size 1099511627776 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: max_xattr_size 65536 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: required_client_features {} 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: last_failure 0 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: last_failure_osd_epoch 0 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: max_mds 1 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: in 0 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: up {0=14494} 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: failed 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: damaged 2024-09-17T10:50:52.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: stopped 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_pools [3] 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: metadata_pool 2 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: inline_data disabled 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: balancer 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: bal_rank_mask -1 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: standby_count_wanted 1 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: qdb_cluster leader: 0 members: 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: [mds.foofs.smithi006.nripkg{0:14494} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.6:6834/1356880066,v1:172.21.15.6:6835/1356880066] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Standby daemons: 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-09-17T10:50:52.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: [mds.foofs.smithi149.hmuwiq{-1:24287} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T10:50:52.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T10:50:52.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T10:50:52.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T10:50:52.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T10:50:52.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-17T10:50:52.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 bash[63946]: 241e5f24b4c7e5602cf542a79e735ae6905975938c64bf0e77276a331f169c1d 2024-09-17T10:50:52.414 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 systemd[1]: Started Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:50:52.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 129684 11.43 MB/sec execute 164 sec latency 44.488 ms 2024-09-17T10:50:52.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:50:52.926 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: pgmap v29: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.1 MiB/s wr, 599 op/s 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006 calling monitor election 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mon.smithi006 is new leader, mons smithi006,smithi149 in quorum (ranks 0,1) 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: monmap epoch 2 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: last_changed 2024-09-17T10:40:19.803100+0000 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: created 2024-09-17T10:38:13.266845+0000 2024-09-17T10:50:52.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: min_mon_release 17 (quincy) 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: election_strategy: 1 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.smithi006 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: 1: [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon.smithi149 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mgrmap e35: smithi006.plhvdm(active, since 54s), standbys: smithi149.ijckag 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: overall HEALTH_OK 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: mgrmap e36: smithi006.plhvdm(active, since 54s), standbys: smithi149.ijckag 2024-09-17T10:50:52.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:52 smithi006 ceph-mon[63965]: from='mgr.14774 ' entity='' 2024-09-17T10:50:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-09-17T10:50:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-09-17T10:50:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: from='mgr.14774 172.21.15.6:0/4087474342' entity='mgr.smithi006.plhvdm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:50:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:50:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: pgmap v29: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.1 MiB/s wr, 599 op/s 2024-09-17T10:50:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: mon.smithi006 calling monitor election 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: mon.smithi006 is new leader, mons smithi006,smithi149 in quorum (ranks 0,1) 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: monmap epoch 2 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: last_changed 2024-09-17T10:40:19.803100+0000 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: created 2024-09-17T10:38:13.266845+0000 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: min_mon_release 17 (quincy) 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: election_strategy: 1 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.smithi006 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: 1: [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon.smithi149 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: mgrmap e35: smithi006.plhvdm(active, since 54s), standbys: smithi149.ijckag 2024-09-17T10:50:53.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: overall HEALTH_OK 2024-09-17T10:50:53.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: mgrmap e36: smithi006.plhvdm(active, since 54s), standbys: smithi149.ijckag 2024-09-17T10:50:53.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:52 smithi149 ceph-mon[26044]: from='mgr.14774 ' entity='' 2024-09-17T10:50:53.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 129949 11.39 MB/sec execute 165 sec latency 48.505 ms 2024-09-17T10:50:54.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 130337 11.37 MB/sec execute 166 sec latency 51.641 ms 2024-09-17T10:50:54.944 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:54 smithi006 ceph-mon[63965]: mgrmap e37: smithi006.plhvdm(active, since 55s), standbys: smithi149.ijckag 2024-09-17T10:50:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:54 smithi149 ceph-mon[26044]: mgrmap e37: smithi006.plhvdm(active, since 55s), standbys: smithi149.ijckag 2024-09-17T10:50:55.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 130636 11.38 MB/sec execute 167 sec latency 41.623 ms 2024-09-17T10:50:56.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 130996 11.41 MB/sec execute 168 sec latency 54.829 ms 2024-09-17T10:50:57.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 131241 11.39 MB/sec execute 169 sec latency 50.925 ms 2024-09-17T10:50:58.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 131531 11.39 MB/sec execute 170 sec latency 49.366 ms 2024-09-17T10:50:59.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:59 smithi006 ceph-mon[63965]: Standby manager daemon smithi149.ijckag restarted 2024-09-17T10:50:59.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:59 smithi006 ceph-mon[63965]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:50:59.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:59 smithi006 ceph-mon[63965]: from='mgr.? 172.21.15.149:0/583489588' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:50:59.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:59 smithi006 ceph-mon[63965]: from='mgr.? 172.21.15.149:0/583489588' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:50:59.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:59 smithi006 ceph-mon[63965]: from='mgr.? 172.21.15.149:0/583489588' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:50:59.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:50:59 smithi006 ceph-mon[63965]: from='mgr.? 172.21.15.149:0/583489588' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:50:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:59 smithi149 ceph-mon[26044]: Standby manager daemon smithi149.ijckag restarted 2024-09-17T10:50:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:59 smithi149 ceph-mon[26044]: Standby manager daemon smithi149.ijckag started 2024-09-17T10:50:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:59 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/583489588' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/crt"}]: dispatch 2024-09-17T10:50:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:59 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/583489588' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T10:50:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:59 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/583489588' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.ijckag/key"}]: dispatch 2024-09-17T10:50:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:50:59 smithi149 ceph-mon[26044]: from='mgr.? 172.21.15.149:0/583489588' entity='mgr.smithi149.ijckag' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T10:50:59.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 131735 11.34 MB/sec execute 171 sec latency 68.245 ms 2024-09-17T10:51:00.365 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: mgrmap e38: smithi006.plhvdm(active, since 60s), standbys: smithi149.ijckag 2024-09-17T10:51:00.365 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: Active manager daemon smithi006.plhvdm restarted 2024-09-17T10:51:00.365 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: Activating manager daemon smithi006.plhvdm 2024-09-17T10:51:00.365 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T10:51:00.365 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: mgrmap e39: smithi006.plhvdm(active, starting, since 0.0069719s), standbys: smithi149.ijckag 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata", "id": "smithi006"} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata", "id": "smithi149"} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mgr metadata", "who": "smithi149.ijckag", "id": "smithi149.ijckag"} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T10:51:00.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata"} : dispatch 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata"} : dispatch 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata"} : dispatch 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: Manager daemon smithi006.plhvdm is now available 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"} : dispatch 2024-09-17T10:51:00.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"} : dispatch 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: mgrmap e38: smithi006.plhvdm(active, since 60s), standbys: smithi149.ijckag 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: Active manager daemon smithi006.plhvdm restarted 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: Activating manager daemon smithi006.plhvdm 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: mgrmap e39: smithi006.plhvdm(active, starting, since 0.0069719s), standbys: smithi149.ijckag 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata", "id": "smithi006"} : dispatch 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata", "id": "smithi149"} : dispatch 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"} : dispatch 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"} : dispatch 2024-09-17T10:51:00.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mgr metadata", "who": "smithi006.plhvdm", "id": "smithi006.plhvdm"} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mgr metadata", "who": "smithi149.ijckag", "id": "smithi149.ijckag"} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata"} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata"} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata"} : dispatch 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: Manager daemon smithi006.plhvdm is now available 2024-09-17T10:51:00.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:51:00.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:51:00.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-17T10:51:00.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/mirror_snapshot_schedule"} : dispatch 2024-09-17T10:51:00.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:00 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.plhvdm/trash_purge_schedule"} : dispatch 2024-09-17T10:51:00.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 131933 11.29 MB/sec execute 172 sec latency 72.121 ms 2024-09-17T10:51:01.635 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:01 smithi006 ceph-mon[63965]: mgrmap e40: smithi006.plhvdm(active, since 1.01376s), standbys: smithi149.ijckag 2024-09-17T10:51:01.636 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:01 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:51:01.636 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:01 smithi006 ceph-mon[63965]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-17T10:51:01.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:01 smithi149 ceph-mon[26044]: mgrmap e40: smithi006.plhvdm(active, since 1.01376s), standbys: smithi149.ijckag 2024-09-17T10:51:01.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:01 smithi149 ceph-mon[26044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T10:51:01.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:01 smithi149 ceph-mon[26044]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-17T10:51:01.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 132126 11.25 MB/sec execute 173 sec latency 48.046 ms 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:02 smithi006 ceph-mon[63965]: [17/Sep/2024:10:51:01] ENGINE Bus STARTING 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:02 smithi006 ceph-mon[63965]: [17/Sep/2024:10:51:01] ENGINE Serving on http://172.21.15.6:8765 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:02 smithi006 ceph-mon[63965]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:02 smithi006 ceph-mon[63965]: [17/Sep/2024:10:51:01] ENGINE Serving on https://172.21.15.6:7150 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:02 smithi006 ceph-mon[63965]: [17/Sep/2024:10:51:01] ENGINE Bus STARTED 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:02 smithi006 ceph-mon[63965]: [17/Sep/2024:10:51:01] ENGINE Error in HTTPServer.serve 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Traceback (most recent call last): 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._connections.run(self.expiration_interval) 2024-09-17T10:51:02.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._run(expiration_interval) 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: s = self.context.wrap_socket( 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: return self.sslsocket_class._create( 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self.do_handshake() 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: self._sslobj.do_handshake() 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T10:51:02.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:02 smithi006 ceph-mon[63965]: mgrmap e41: smithi006.plhvdm(active, since 2s), standbys: smithi149.ijckag 2024-09-17T10:51:02.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 132378 11.20 MB/sec execute 174 sec latency 62.645 ms 2024-09-17T10:51:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:51:01] ENGINE Bus STARTING 2024-09-17T10:51:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:51:01] ENGINE Serving on http://172.21.15.6:8765 2024-09-17T10:51:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:02 smithi149 ceph-mon[26044]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:51:01] ENGINE Serving on https://172.21.15.6:7150 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:51:01] ENGINE Bus STARTED 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:02 smithi149 ceph-mon[26044]: [17/Sep/2024:10:51:01] ENGINE Error in HTTPServer.serve 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Traceback (most recent call last): 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._connections.run(self.expiration_interval) 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._run(expiration_interval) 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T10:51:02.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: s = self.context.wrap_socket( 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: return self.sslsocket_class._create( 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self.do_handshake() 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: self._sslobj.do_handshake() 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T10:51:02.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:02 smithi149 ceph-mon[26044]: mgrmap e41: smithi006.plhvdm(active, since 2s), standbys: smithi149.ijckag 2024-09-17T10:51:03.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 132717 11.18 MB/sec execute 175 sec latency 63.438 ms 2024-09-17T10:51:04.511 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:04 smithi006 ceph-mon[63965]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-17T10:51:04.512 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:04 smithi006 ceph-mon[63965]: mgrmap e42: smithi006.plhvdm(active, since 4s), standbys: smithi149.ijckag 2024-09-17T10:51:04.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 132998 11.15 MB/sec execute 176 sec latency 60.206 ms 2024-09-17T10:51:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:04 smithi149 ceph-mon[26044]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-17T10:51:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:04 smithi149 ceph-mon[26044]: mgrmap e42: smithi006.plhvdm(active, since 4s), standbys: smithi149.ijckag 2024-09-17T10:51:05.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:05 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:05.732 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:05 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:05.732 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:05 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:05.732 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:05 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:05.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 133395 11.15 MB/sec execute 177 sec latency 52.611 ms 2024-09-17T10:51:05.887 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:05 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:05.887 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:05 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:05.887 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:05 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:05.887 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:05 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:06.765 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:06 smithi006 ceph-mon[63965]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-17T10:51:06.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 133678 11.11 MB/sec execute 178 sec latency 59.466 ms 2024-09-17T10:51:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:06 smithi149 ceph-mon[26044]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-17T10:51:07.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 133988 11.11 MB/sec execute 179 sec latency 82.724 ms 2024-09-17T10:51:08.040 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:51:08.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 134264 11.11 MB/sec execute 180 sec latency 117.968 ms 2024-09-17T10:51:08.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:08 smithi006 ceph-mon[63965]: pgmap v7: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 2.4 MiB/s wr, 320 op/s 2024-09-17T10:51:08.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:08.839 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:08.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:08 smithi149 ceph-mon[26044]: pgmap v7: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 2.4 MiB/s wr, 320 op/s 2024-09-17T10:51:08.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:08 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:08.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:08 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:09.000 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:51:09.000 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (2m) 0s ago 11m 18.5M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:51:09.000 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (11m) 0s ago 11m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:51:09.000 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (10m) 4s ago 10m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:51:09.000 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (2m) 0s ago 11m 42.1M - dad864ee21e9 89df198e1a9b 2024-09-17T10:51:09.000 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (5m) 0s ago 5m 37.4M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:51:09.000 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (5m) 4s ago 5m 13.9M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:51:09.000 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (90s) 0s ago 12m 488M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (35s) 4s ago 10m 451M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (17s) 0s ago 12m 32.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (10m) 4s ago 10m 49.0M 2048M 17.2.0 e1d6a67b021e a823656bce1e 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (5m) 0s ago 5m 276M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (5m) 4s ago 5m 85.4M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (2m) 0s ago 11m 9675k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (100s) 4s ago 10m 10.4M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (10m) 0s ago 10m 480M 1937M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (9m) 0s ago 9m 399M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (9m) 0s ago 9m 309M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (8m) 0s ago 8m 443M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (8m) 4s ago 8m 468M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (7m) 4s ago 7m 514M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:51:09.001 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (7m) 4s ago 7m 428M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:51:09.002 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (7m) 4s ago 7m 576M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:51:09.002 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (42s) 0s ago 10m 44.1M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:51:09.604 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:51:09.604 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:51:09.604 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:51:09.605 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:51:09.606 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:51:09.606 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-17T10:51:09.606 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 3 2024-09-17T10:51:09.606 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:51:09.606 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:51:09.808 INFO:teuthology.orchestra.run.smithi006.stdout: 5 134602 11.13 MB/sec execute 181 sec latency 47.350 ms 2024-09-17T10:51:09.856 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:09 smithi006 ceph-mon[63965]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:09.856 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:09 smithi006 ceph-mon[63965]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:09.856 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:09 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:09.856 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:09 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:09.856 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:09 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1951988469' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:51:09.856 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:09 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:09.856 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:09 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:09.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:09 smithi149 ceph-mon[26044]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:09.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:09 smithi149 ceph-mon[26044]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:09.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:09 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:09.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:09 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:09.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:09 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1951988469' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:51:09.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:09 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:09.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:09 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.125 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:51:10.125 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:51:10.125 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:51:10.125 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:51:10.125 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:51:10.125 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:51:10.126 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:51:10.126 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "3/23 daemons upgraded", 2024-09-17T10:51:10.126 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-09-17T10:51:10.126 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:51:10.126 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:51:10.720 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:51:10.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 134842 11.11 MB/sec execute 182 sec latency 52.232 ms 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: pgmap v8: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 1.8 MiB/s wr, 248 op/s 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"} : dispatch 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: Updating smithi149:/etc/ceph/ceph.conf 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:51:10.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.890 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.890 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.890 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.890 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.890 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: pgmap v8: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 1.8 MiB/s wr, 248 op/s 2024-09-17T10:51:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"} : dispatch 2024-09-17T10:51:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:51:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: Updating smithi006:/etc/ceph/ceph.conf 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: Updating smithi149:/etc/ceph/ceph.conf 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:10.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:10 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:11.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:51:11.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:51:11.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:11.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:51:11.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:51:11.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1998836867' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:51:11.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:51:11.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:51:11.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:11 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "quorum_status"} : dispatch 2024-09-17T10:51:11.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 135086 11.09 MB/sec execute 183 sec latency 73.321 ms 2024-09-17T10:51:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:51:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-09-17T10:51:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: Updating smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:51:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: Updating smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T10:51:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: from='client.? 172.21.15.6:0/1998836867' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:51:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:51:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:51:11.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:11 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "quorum_status"} : dispatch 2024-09-17T10:51:12.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:12 smithi006 ceph-mon[63965]: pgmap v9: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-09-17T10:51:12.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:12 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:12.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:12 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T10:51:12.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:12 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T10:51:12.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:12 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:12.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 135339 11.06 MB/sec execute 184 sec latency 53.177 ms 2024-09-17T10:51:12.918 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:12 smithi149 ceph-mon[26044]: pgmap v9: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-09-17T10:51:12.918 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:12 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:12.918 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:12 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T10:51:12.919 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:12 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T10:51:12.919 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:12 smithi149 ceph-mon[26044]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:13.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:13 smithi006 ceph-mon[63965]: Upgrade: Updating mon.smithi149 2024-09-17T10:51:13.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:13 smithi006 ceph-mon[63965]: Deploying daemon mon.smithi149 on smithi149 2024-09-17T10:51:13.794 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:13 smithi149 ceph-mon[26044]: Upgrade: Updating mon.smithi149 2024-09-17T10:51:13.795 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:13 smithi149 ceph-mon[26044]: Deploying daemon mon.smithi149 on smithi149 2024-09-17T10:51:13.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 135614 11.03 MB/sec execute 185 sec latency 47.998 ms 2024-09-17T10:51:14.361 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:14 smithi149 systemd[1]: Stopping Ceph mon.smithi149 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:51:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:14 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149[26026]: 2024-09-17T10:51:14.358+0000 7f2f2a85c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi149 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:51:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:14 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149[26026]: 2024-09-17T10:51:14.358+0000 7f2f2a85c700 -1 mon.smithi149@1(peon) e2 *** Got Signal Terminated *** 2024-09-17T10:51:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:14 smithi149 podman[49538]: 2024-09-17 10:51:14.500724463 +0000 UTC m=+0.251658211 container died a823656bce1e1bbe03b270f4044697515103f7711013a9267e709c100877a7f5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , ceph=True, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_CLEAN=True, architecture=x86_64, distribution-scope=public, release=754, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8) 2024-09-17T10:51:14.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 135957 11.00 MB/sec execute 186 sec latency 42.895 ms 2024-09-17T10:51:15.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 podman[49538]: 2024-09-17 10:51:15.066977912 +0000 UTC m=+0.817911658 container cleanup a823656bce1e1bbe03b270f4044697515103f7711013a9267e709c100877a7f5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, name=centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , architecture=x86_64, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vcs-type=git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc.) 2024-09-17T10:51:15.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 bash[49538]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149 2024-09-17T10:51:15.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 podman[49550]: 2024-09-17 10:51:15.327169919 +0000 UTC m=+0.829346984 container remove a823656bce1e1bbe03b270f4044697515103f7711013a9267e709c100877a7f5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.component=centos-stream-container, distribution-scope=public, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, RELEASE=HEAD) 2024-09-17T10:51:15.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi149.service: Deactivated successfully. 2024-09-17T10:51:15.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 systemd[1]: Stopped Ceph mon.smithi149 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:51:15.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi149.service: Consumed 6.974s CPU time. 2024-09-17T10:51:15.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 136368 10.99 MB/sec execute 187 sec latency 54.027 ms 2024-09-17T10:51:16.063 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 systemd[1]: Starting Ceph mon.smithi149 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:51:16.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 podman[49678]: 2024-09-17 10:51:15.840248886 +0000 UTC m=+0.017562257 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:51:16.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:15 smithi149 podman[49678]: 2024-09-17 10:51:15.9450119 +0000 UTC m=+0.122325258 container create 800698b81fca4d586e7cd9100db5ebe3043010a02b963da8fa06e50ccb5c88e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T10:51:16.316 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 podman[49678]: 2024-09-17 10:51:16.190292035 +0000 UTC m=+0.367605392 container init 800698b81fca4d586e7cd9100db5ebe3043010a02b963da8fa06e50ccb5c88e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T10:51:16.316 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 podman[49678]: 2024-09-17 10:51:16.193983679 +0000 UTC m=+0.371297036 container start 800698b81fca4d586e7cd9100db5ebe3043010a02b963da8fa06e50ccb5c88e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T10:51:16.316 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T10:51:16.316 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev), process ceph-mon, pid 2 2024-09-17T10:51:16.316 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: pidfile_write: ignore empty --pid-file 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: load: jerasure load: lrc 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: RocksDB version: 7.9.2 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Git sha 0 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Compile date 2024-09-15 20:52:36 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: DB SUMMARY 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: DB Session ID: 4NJ7VISW038HMLUE1L7W 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: CURRENT file: CURRENT 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi149/store.db dir, Total Num: 1, files: 000024.sst 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi149/store.db: 000022.log size: 7127985 ; 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.error_if_exists: 0 2024-09-17T10:51:16.317 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.create_if_missing: 0 2024-09-17T10:51:16.318 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.paranoid_checks: 1 2024-09-17T10:51:16.318 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T10:51:16.318 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T10:51:16.318 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T10:51:16.318 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.env: 0x560ab8550120 2024-09-17T10:51:16.318 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T10:51:16.318 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.info_log: 0x560abab458e0 2024-09-17T10:51:16.318 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T10:51:16.319 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.statistics: (nil) 2024-09-17T10:51:16.319 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.use_fsync: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_log_file_size: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.allow_fallocate: 1 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.use_direct_reads: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.db_log_dir: 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.wal_dir: 2024-09-17T10:51:16.320 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.write_buffer_manager: 0x560abab49b80 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T10:51:16.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.unordered_write: 0 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.row_cache: None 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.wal_filter: None 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.two_write_queues: 0 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T10:51:16.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.wal_compression: 0 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.atomic_flush: 0 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.log_readahead_size: 0 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_background_jobs: 2 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_background_compactions: -1 2024-09-17T10:51:16.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_subcompactions: 1 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_open_files: -1 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_background_flushes: -1 2024-09-17T10:51:16.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Compression algorithms supported: 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: kZSTD supported: 0 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: kXpressCompression supported: 0 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: kBZip2Compression supported: 0 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: kLZ4Compression supported: 1 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: kZlibCompression supported: 1 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: kSnappyCompression supported: 1 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi149/store.db/MANIFEST-000009 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T10:51:16.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.merge_operator: 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_filter: None 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560abab45540) 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: cache_index_and_filter_blocks: 1 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: pin_top_level_index_and_filter: 1 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: index_type: 0 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: data_block_index_type: 0 2024-09-17T10:51:16.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: index_shortening: 1 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: checksum: 4 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: no_block_cache: 0 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_cache: 0x560abab39610 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_cache_name: BinnedLRUCache 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_cache_options: 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: capacity : 536870912 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: num_shard_bits : 4 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: strict_capacity_limit : 0 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: high_pri_pool_ratio: 0.000 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_cache_compressed: (nil) 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: persistent_cache: (nil) 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_size: 4096 2024-09-17T10:51:16.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_size_deviation: 10 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_restart_interval: 16 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: index_block_restart_interval: 1 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: metadata_block_size: 4096 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: partition_filters: 0 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: use_delta_encoding: 1 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: filter_policy: bloomfilter 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: whole_key_filtering: 1 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: verify_compression: 0 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: read_amp_bytes_per_bit: 0 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: format_version: 5 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: enable_index_compression: 1 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_align: 0 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: max_auto_readahead_size: 262144 2024-09-17T10:51:16.328 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: prepopulate_block_cache: 0 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: initial_auto_readahead_size: 8192 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression: NoCompression 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.num_levels: 7 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T10:51:16.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T10:51:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T10:51:16.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T10:51:16.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.inplace_update_support: 0 2024-09-17T10:51:16.333 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.bloom_locality: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.max_successive_merges: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.ttl: 2592000 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T10:51:16.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.enable_blob_files: false 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.min_blob_size: 0 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T10:51:16.335 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi149/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12267, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fbac8b5c-708e-4dbc-8e5f-6a3d2d8b2556 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726570276224712, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726570276243032, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4339271, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12272, "largest_seqno": 13955, "table_properties": {"data_size": 4331634, "index_size": 4864, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15753, "raw_average_key_size": 23, "raw_value_size": 4316880, "raw_average_value_size": 6367, "num_data_blocks": 220, "num_entries": 678, "num_filter_entries": 678, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726570276, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fbac8b5c-708e-4dbc-8e5f-6a3d2d8b2556", "db_session_id": "4NJ7VISW038HMLUE1L7W", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726570276243172, "job": 1, "event": "recovery_finished"} 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi149/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560abab68e00 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: DB pointer 0x560abac6a000 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T10:51:16.336 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ** DB Stats ** 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ** Compaction Stats [default] ** 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T10:51:16.337 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: L0 1/0 4.14 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: L6 1/0 6.78 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Sum 2/0 10.91 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ** Compaction Stats [default] ** 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 492.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-17T10:51:16.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Cumulative compaction: 0.00 GB write, 196.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Interval compaction: 0.00 GB write, 196.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Block cache BinnedLRUCache@0x560abab39610#2 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: starting mon.smithi149 rank 1 at public addrs [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] at bind addrs [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi149 fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: mon.smithi149@-1(???) e2 preinit fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: mon.smithi149@-1(???).mds e7 new map 2024-09-17T10:51:16.339 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: mon.smithi149@-1(???).mds e7 print_map 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: e7 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: legacy client fscid: 1 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Filesystem 'foofs' (1) 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: fs_name foofs 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: epoch 5 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: created 2024-09-17T10:45:30.960183+0000 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: modified 2024-09-17T10:45:36.993299+0000 2024-09-17T10:51:16.340 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: tableserver 0 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: root 0 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: session_timeout 60 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: session_autoclose 300 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: max_file_size 1099511627776 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: max_xattr_size 65536 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: required_client_features {} 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: last_failure 0 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: last_failure_osd_epoch 0 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: max_mds 1 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: in 0 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: up {0=14494} 2024-09-17T10:51:16.341 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: failed 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: damaged 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: stopped 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: data_pools [3] 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: metadata_pool 2 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: inline_data disabled 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: balancer 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: bal_rank_mask -1 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: standby_count_wanted 1 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: qdb_cluster leader: 0 members: 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: [mds.foofs.smithi006.nripkg{0:14494} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.6:6834/1356880066,v1:172.21.15.6:6835/1356880066] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Standby daemons: 2024-09-17T10:51:16.342 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-09-17T10:51:16.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: [mds.foofs.smithi149.hmuwiq{-1:24287} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T10:51:16.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: mon.smithi149@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T10:51:16.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: mon.smithi149@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T10:51:16.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: mon.smithi149@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T10:51:16.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: mon.smithi149@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T10:51:16.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 ceph-mon[49697]: mon.smithi149@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-17T10:51:16.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 bash[49678]: 800698b81fca4d586e7cd9100db5ebe3043010a02b963da8fa06e50ccb5c88e3 2024-09-17T10:51:16.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:16 smithi149 systemd[1]: Started Ceph mon.smithi149 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:51:16.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 136779 10.97 MB/sec execute 188 sec latency 54.122 ms 2024-09-17T10:51:17.775 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:17 smithi006 ceph-mon[63965]: pgmap v10: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.4 MiB/s wr, 355 op/s 2024-09-17T10:51:17.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 137147 10.97 MB/sec execute 189 sec latency 58.653 ms 2024-09-17T10:51:17.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:17 smithi149 ceph-mon[49697]: pgmap v10: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.4 MiB/s wr, 355 op/s 2024-09-17T10:51:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.2 MiB/s wr, 544 op/s 2024-09-17T10:51:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: overall HEALTH_OK 2024-09-17T10:51:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: mon.smithi006 calling monitor election 2024-09-17T10:51:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: mon.smithi149 calling monitor election 2024-09-17T10:51:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata", "id": "smithi006"} : dispatch 2024-09-17T10:51:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata", "id": "smithi149"} : dispatch 2024-09-17T10:51:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: mon.smithi006 is new leader, mons smithi006,smithi149 in quorum (ranks 0,1) 2024-09-17T10:51:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: monmap epoch 3 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: last_changed 2024-09-17T10:51:17.689635+0000 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: created 2024-09-17T10:38:13.266845+0000 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: min_mon_release 19 (squid) 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: election_strategy: 1 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.smithi006 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: 1: [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon.smithi149 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: mgrmap e42: smithi006.plhvdm(active, since 18s), standbys: smithi149.ijckag 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: overall HEALTH_OK 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:18.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:18 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:51:18.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 137531 10.96 MB/sec execute 190 sec latency 70.383 ms 2024-09-17T10:51:18.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.2 MiB/s wr, 544 op/s 2024-09-17T10:51:18.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: overall HEALTH_OK 2024-09-17T10:51:18.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: mon.smithi006 calling monitor election 2024-09-17T10:51:18.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: mon.smithi149 calling monitor election 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata", "id": "smithi006"} : dispatch 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mon metadata", "id": "smithi149"} : dispatch 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: mon.smithi006 is new leader, mons smithi006,smithi149 in quorum (ranks 0,1) 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: monmap epoch 3 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: fsid ddf74116-74e0-11ef-bceb-c7b262605968 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: last_changed 2024-09-17T10:51:17.689635+0000 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: created 2024-09-17T10:38:13.266845+0000 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: min_mon_release 19 (squid) 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: election_strategy: 1 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.smithi006 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: 1: [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon.smithi149 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T10:51:18.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: mgrmap e42: smithi006.plhvdm(active, since 18s), standbys: smithi149.ijckag 2024-09-17T10:51:18.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: overall HEALTH_OK 2024-09-17T10:51:18.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:18.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:18.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:18 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:51:19.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 138017 11.02 MB/sec execute 191 sec latency 51.033 ms 2024-09-17T10:51:20.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 138330 11.02 MB/sec execute 192 sec latency 40.492 ms 2024-09-17T10:51:20.849 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:20 smithi006 ceph-mon[63965]: pgmap v13: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 358 op/s 2024-09-17T10:51:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:20 smithi149 ceph-mon[49697]: pgmap v13: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 358 op/s 2024-09-17T10:51:21.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 138733 11.03 MB/sec execute 193 sec latency 42.990 ms 2024-09-17T10:51:22.615 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:22 smithi006 ceph-mon[63965]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.3 MiB/s wr, 589 op/s 2024-09-17T10:51:22.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:22.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:22.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:22.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:22.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 139058 11.00 MB/sec execute 194 sec latency 72.288 ms 2024-09-17T10:51:22.841 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:22 smithi149 ceph-mon[49697]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.3 MiB/s wr, 589 op/s 2024-09-17T10:51:22.841 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:22.841 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:22.841 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:22.841 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:23.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 139353 10.97 MB/sec execute 195 sec latency 94.884 ms 2024-09-17T10:51:24.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:24 smithi006 ceph-mon[63965]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 419 op/s 2024-09-17T10:51:24.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 139770 10.96 MB/sec execute 196 sec latency 58.008 ms 2024-09-17T10:51:24.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:24 smithi149 ceph-mon[49697]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 419 op/s 2024-09-17T10:51:25.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 140083 10.93 MB/sec execute 197 sec latency 43.385 ms 2024-09-17T10:51:26.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 419 op/s 2024-09-17T10:51:26.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:26.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:26.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:26.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:51:26.695 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:26.695 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:26.695 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: Reconfiguring mon.smithi006 (monmap changed)... 2024-09-17T10:51:26.695 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T10:51:26.695 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T10:51:26.695 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:26.695 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:26 smithi006 ceph-mon[63965]: Reconfiguring daemon mon.smithi006 on smithi006 2024-09-17T10:51:26.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 140547 10.94 MB/sec execute 198 sec latency 50.067 ms 2024-09-17T10:51:26.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 419 op/s 2024-09-17T10:51:26.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:26.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:26.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:26.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:51:26.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:26.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:26.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: Reconfiguring mon.smithi006 (monmap changed)... 2024-09-17T10:51:26.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T10:51:26.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T10:51:26.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:26.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:26 smithi149 ceph-mon[49697]: Reconfiguring daemon mon.smithi006 on smithi006 2024-09-17T10:51:27.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 140944 10.92 MB/sec execute 199 sec latency 71.724 ms 2024-09-17T10:51:28.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:28 smithi006 ceph-mon[63965]: pgmap v17: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 646 op/s 2024-09-17T10:51:28.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:28.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:28.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi006.plhvdm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T10:51:28.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T10:51:28.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:28.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 141278 10.94 MB/sec execute 200 sec latency 58.619 ms 2024-09-17T10:51:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:28 smithi149 ceph-mon[49697]: pgmap v17: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 646 op/s 2024-09-17T10:51:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi006.plhvdm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T10:51:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T10:51:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:29.737 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:29 smithi006 ceph-mon[63965]: Reconfiguring mgr.smithi006.plhvdm (monmap changed)... 2024-09-17T10:51:29.738 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:29 smithi006 ceph-mon[63965]: Reconfiguring daemon mgr.smithi006.plhvdm on smithi006 2024-09-17T10:51:29.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 141695 10.97 MB/sec execute 201 sec latency 48.424 ms 2024-09-17T10:51:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:29 smithi149 ceph-mon[49697]: Reconfiguring mgr.smithi006.plhvdm (monmap changed)... 2024-09-17T10:51:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:29 smithi149 ceph-mon[49697]: Reconfiguring daemon mgr.smithi006.plhvdm on smithi006 2024-09-17T10:51:30.756 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:30 smithi006 ceph-mon[63965]: pgmap v18: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 457 op/s 2024-09-17T10:51:30.756 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:51:30.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 142074 10.97 MB/sec execute 202 sec latency 40.307 ms 2024-09-17T10:51:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:30 smithi149 ceph-mon[49697]: pgmap v18: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 457 op/s 2024-09-17T10:51:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:51:31.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 142410 10.97 MB/sec execute 203 sec latency 41.723 ms 2024-09-17T10:51:32.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 142703 10.94 MB/sec execute 204 sec latency 42.075 ms 2024-09-17T10:51:33.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:32 smithi006 ceph-mon[63965]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.2 MiB/s wr, 681 op/s 2024-09-17T10:51:33.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:33.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:33.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:32 smithi006 ceph-mon[63965]: Reconfiguring crash.smithi006 (monmap changed)... 2024-09-17T10:51:33.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T10:51:33.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:33.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:32 smithi006 ceph-mon[63965]: Reconfiguring daemon crash.smithi006 on smithi006 2024-09-17T10:51:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:32 smithi149 ceph-mon[49697]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.2 MiB/s wr, 681 op/s 2024-09-17T10:51:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:32 smithi149 ceph-mon[49697]: Reconfiguring crash.smithi006 (monmap changed)... 2024-09-17T10:51:33.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T10:51:33.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:33.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:32 smithi149 ceph-mon[49697]: Reconfiguring daemon crash.smithi006 on smithi006 2024-09-17T10:51:33.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 143089 10.91 MB/sec execute 205 sec latency 65.570 ms 2024-09-17T10:51:34.800 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:34 smithi006 ceph-mon[63965]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 450 op/s 2024-09-17T10:51:34.800 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:34.800 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:34.800 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T10:51:34.800 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:34.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 143452 10.90 MB/sec execute 206 sec latency 59.017 ms 2024-09-17T10:51:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:34 smithi149 ceph-mon[49697]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 450 op/s 2024-09-17T10:51:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T10:51:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:35.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 143744 10.87 MB/sec execute 207 sec latency 58.731 ms 2024-09-17T10:51:35.816 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:35 smithi006 ceph-mon[63965]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T10:51:35.816 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:35 smithi006 ceph-mon[63965]: Reconfiguring daemon osd.0 on smithi006 2024-09-17T10:51:36.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:35 smithi149 ceph-mon[49697]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T10:51:36.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:35 smithi149 ceph-mon[49697]: Reconfiguring daemon osd.0 on smithi006 2024-09-17T10:51:36.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 144081 10.88 MB/sec execute 208 sec latency 66.172 ms 2024-09-17T10:51:36.874 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:36 smithi006 ceph-mon[63965]: pgmap v21: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.7 MiB/s wr, 452 op/s 2024-09-17T10:51:37.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:36 smithi149 ceph-mon[49697]: pgmap v21: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.7 MiB/s wr, 452 op/s 2024-09-17T10:51:37.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 144434 10.86 MB/sec execute 209 sec latency 77.861 ms 2024-09-17T10:51:38.650 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:38.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:38.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:38 smithi006 ceph-mon[63965]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T10:51:38.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T10:51:38.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:38.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:38 smithi006 ceph-mon[63965]: Reconfiguring daemon osd.1 on smithi006 2024-09-17T10:51:38.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:38 smithi006 ceph-mon[63965]: pgmap v22: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 673 op/s 2024-09-17T10:51:38.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:38.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:38.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:38 smithi149 ceph-mon[49697]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T10:51:38.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T10:51:38.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:38.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:38 smithi149 ceph-mon[49697]: Reconfiguring daemon osd.1 on smithi006 2024-09-17T10:51:38.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:38 smithi149 ceph-mon[49697]: pgmap v22: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 673 op/s 2024-09-17T10:51:38.809 INFO:teuthology.orchestra.run.smithi006.stdout: 5 144787 10.87 MB/sec execute 210 sec latency 49.663 ms 2024-09-17T10:51:39.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 145147 10.88 MB/sec execute 211 sec latency 60.582 ms 2024-09-17T10:51:40.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:40 smithi006 ceph-mon[63965]: pgmap v23: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-09-17T10:51:40.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:40.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:40.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T10:51:40.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:40.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 145556 10.90 MB/sec execute 212 sec latency 33.056 ms 2024-09-17T10:51:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:40 smithi149 ceph-mon[49697]: pgmap v23: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-09-17T10:51:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T10:51:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:41.305 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:51:41.705 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:41 smithi006 ceph-mon[63965]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T10:51:41.706 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:41 smithi006 ceph-mon[63965]: Reconfiguring daemon osd.2 on smithi006 2024-09-17T10:51:41.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 145904 10.90 MB/sec execute 213 sec latency 75.889 ms 2024-09-17T10:51:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:41 smithi149 ceph-mon[49697]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T10:51:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:41 smithi149 ceph-mon[49697]: Reconfiguring daemon osd.2 on smithi006 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (3m) 33s ago 12m 18.5M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (12m) 33s ago 12m 6405k - 17.2.0 e1d6a67b021e 665e53259c83 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (11m) 20s ago 11m 6400k - 17.2.0 e1d6a67b021e 9bd415a6d2cc 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (2m) 33s ago 11m 42.1M - dad864ee21e9 89df198e1a9b 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (6m) 33s ago 6m 37.4M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (6m) 20s ago 6m 14.5M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (2m) 33s ago 13m 488M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (69s) 20s ago 11m 452M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:51:42.506 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (51s) 33s ago 13m 32.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (26s) 20s ago 11m 23.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (5m) 33s ago 5m 276M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (5m) 20s ago 5m 85.9M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (2m) 33s ago 11m 9675k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (2m) 20s ago 11m 12.4M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (10m) 33s ago 10m 480M 1937M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (10m) 33s ago 10m 399M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (9m) 33s ago 9m 309M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (9m) 33s ago 9m 443M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (8m) 20s ago 8m 473M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (8m) 20s ago 8m 538M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (8m) 20s ago 8m 458M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:51:42.507 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (7m) 20s ago 7m 328M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:51:42.508 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (76s) 33s ago 11m 44.1M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:51:42.730 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:42 smithi006 ceph-mon[63965]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:42.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:42 smithi006 ceph-mon[63965]: pgmap v24: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.8 MiB/s wr, 648 op/s 2024-09-17T10:51:42.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:42 smithi006 ceph-mon[63965]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:42.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:42.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:42.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T10:51:42.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:42.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 146192 10.88 MB/sec execute 214 sec latency 54.564 ms 2024-09-17T10:51:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:42 smithi149 ceph-mon[49697]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:42 smithi149 ceph-mon[49697]: pgmap v24: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.8 MiB/s wr, 648 op/s 2024-09-17T10:51:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:42 smithi149 ceph-mon[49697]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T10:51:42.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:43.240 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:51:43.240 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:51:43.241 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:51:43.242 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T10:51:43.242 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T10:51:43.242 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:51:43.242 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:51:43.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:43 smithi006 ceph-mon[63965]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T10:51:43.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:43 smithi006 ceph-mon[63965]: Reconfiguring daemon osd.3 on smithi006 2024-09-17T10:51:43.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:43 smithi006 ceph-mon[63965]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:43.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:43 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3275444729' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:51:43.775 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "4/23 daemons upgraded", 2024-09-17T10:51:43.776 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T10:51:43.776 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:51:43.776 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:51:43.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 146509 10.84 MB/sec execute 215 sec latency 141.063 ms 2024-09-17T10:51:43.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:43 smithi149 ceph-mon[49697]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T10:51:43.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:43 smithi149 ceph-mon[49697]: Reconfiguring daemon osd.3 on smithi006 2024-09-17T10:51:43.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:43 smithi149 ceph-mon[49697]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:43.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:43 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3275444729' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:51:44.500 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:51:44.775 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:44 smithi006 ceph-mon[63965]: pgmap v25: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-17T10:51:44.776 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:44 smithi006 ceph-mon[63965]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:44.776 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:44 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3428039820' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:51:44.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 146866 10.82 MB/sec execute 216 sec latency 44.759 ms 2024-09-17T10:51:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:44 smithi149 ceph-mon[49697]: pgmap v25: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-17T10:51:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:44 smithi149 ceph-mon[49697]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:51:44.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:44 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3428039820' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:51:45.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:45.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:45.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:45 smithi006 ceph-mon[63965]: Reconfiguring mds.foofs.smithi006.nripkg (monmap changed)... 2024-09-17T10:51:45.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.nripkg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T10:51:45.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:45.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:45 smithi006 ceph-mon[63965]: Reconfiguring daemon mds.foofs.smithi006.nripkg on smithi006 2024-09-17T10:51:45.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:51:45.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 147208 10.81 MB/sec execute 217 sec latency 52.063 ms 2024-09-17T10:51:46.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:46.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:46.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:45 smithi149 ceph-mon[49697]: Reconfiguring mds.foofs.smithi006.nripkg (monmap changed)... 2024-09-17T10:51:46.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.nripkg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T10:51:46.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:46.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:45 smithi149 ceph-mon[49697]: Reconfiguring daemon mds.foofs.smithi006.nripkg on smithi006 2024-09-17T10:51:46.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:51:46.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 147631 10.82 MB/sec execute 218 sec latency 54.005 ms 2024-09-17T10:51:46.812 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:46 smithi006 ceph-mon[63965]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 524 op/s 2024-09-17T10:51:47.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:46 smithi149 ceph-mon[49697]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 524 op/s 2024-09-17T10:51:47.810 INFO:teuthology.orchestra.run.smithi006.stdout: 5 148053 10.81 MB/sec execute 219 sec latency 35.949 ms 2024-09-17T10:51:48.082 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:48 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:48.082 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:48 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:48.082 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:48 smithi006 ceph-mon[63965]: Reconfiguring crash.smithi149 (monmap changed)... 2024-09-17T10:51:48.082 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:48 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T10:51:48.082 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:48 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:48.082 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:48 smithi006 ceph-mon[63965]: Reconfiguring daemon crash.smithi149 on smithi149 2024-09-17T10:51:48.082 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:48 smithi006 ceph-mon[63965]: pgmap v27: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.3 MiB/s wr, 639 op/s 2024-09-17T10:51:48.398 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:48 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:48.398 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:48 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:48.398 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:48 smithi149 ceph-mon[49697]: Reconfiguring crash.smithi149 (monmap changed)... 2024-09-17T10:51:48.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:48 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T10:51:48.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:48 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:48.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:48 smithi149 ceph-mon[49697]: Reconfiguring daemon crash.smithi149 on smithi149 2024-09-17T10:51:48.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:48 smithi149 ceph-mon[49697]: pgmap v27: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.3 MiB/s wr, 639 op/s 2024-09-17T10:51:48.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 148403 10.82 MB/sec execute 220 sec latency 58.755 ms 2024-09-17T10:51:49.605 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:49 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:49.606 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:49 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:49.606 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:49 smithi006 ceph-mon[63965]: Reconfiguring mgr.smithi149.ijckag (monmap changed)... 2024-09-17T10:51:49.606 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:49 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T10:51:49.606 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:49 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T10:51:49.606 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:49 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:49.606 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:49 smithi006 ceph-mon[63965]: Reconfiguring daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:51:49.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:49 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:49.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:49 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:49.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:49 smithi149 ceph-mon[49697]: Reconfiguring mgr.smithi149.ijckag (monmap changed)... 2024-09-17T10:51:49.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:49 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi149.ijckag", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T10:51:49.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:49 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T10:51:49.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:49 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:49.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:49 smithi149 ceph-mon[49697]: Reconfiguring daemon mgr.smithi149.ijckag on smithi149 2024-09-17T10:51:49.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 148826 10.85 MB/sec execute 221 sec latency 30.637 ms 2024-09-17T10:51:50.578 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:50 smithi149 ceph-mon[49697]: pgmap v28: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-17T10:51:50.578 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:50.578 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:50.578 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T10:51:50.578 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T10:51:50.578 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:50.618 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:50 smithi006 ceph-mon[63965]: pgmap v28: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-17T10:51:50.618 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:50.618 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:50.618 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T10:51:50.618 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T10:51:50.618 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:50.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 149186 10.85 MB/sec execute 222 sec latency 44.645 ms 2024-09-17T10:51:51.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:51 smithi006 ceph-mon[63965]: Reconfiguring mon.smithi149 (monmap changed)... 2024-09-17T10:51:51.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:51 smithi006 ceph-mon[63965]: Reconfiguring daemon mon.smithi149 on smithi149 2024-09-17T10:51:51.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:51 smithi149 ceph-mon[49697]: Reconfiguring mon.smithi149 (monmap changed)... 2024-09-17T10:51:51.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:51 smithi149 ceph-mon[49697]: Reconfiguring daemon mon.smithi149 on smithi149 2024-09-17T10:51:51.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 149697 10.85 MB/sec execute 223 sec latency 36.285 ms 2024-09-17T10:51:52.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 150363 10.86 MB/sec execute 224 sec latency 25.461 ms 2024-09-17T10:51:52.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:52 smithi006 ceph-mon[63965]: pgmap v29: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.8 MiB/s wr, 644 op/s 2024-09-17T10:51:52.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:52.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:52.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:52 smithi006 ceph-mon[63965]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T10:51:52.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T10:51:52.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:52.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:52 smithi006 ceph-mon[63965]: Reconfiguring daemon osd.4 on smithi149 2024-09-17T10:51:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:52 smithi149 ceph-mon[49697]: pgmap v29: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.8 MiB/s wr, 644 op/s 2024-09-17T10:51:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:52 smithi149 ceph-mon[49697]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T10:51:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T10:51:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:53.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:52 smithi149 ceph-mon[49697]: Reconfiguring daemon osd.4 on smithi149 2024-09-17T10:51:53.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 150953 10.87 MB/sec execute 225 sec latency 32.384 ms 2024-09-17T10:51:54.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:54.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:54.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:54 smithi006 ceph-mon[63965]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T10:51:54.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T10:51:54.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:54.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:54 smithi006 ceph-mon[63965]: Reconfiguring daemon osd.5 on smithi149 2024-09-17T10:51:54.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:54 smithi006 ceph-mon[63965]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-17T10:51:54.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:54.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:54.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:54 smithi149 ceph-mon[49697]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T10:51:54.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T10:51:54.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:54.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:54 smithi149 ceph-mon[49697]: Reconfiguring daemon osd.5 on smithi149 2024-09-17T10:51:54.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:54 smithi149 ceph-mon[49697]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-17T10:51:54.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 151482 10.87 MB/sec execute 226 sec latency 50.590 ms 2024-09-17T10:51:55.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 151975 10.91 MB/sec execute 227 sec latency 34.548 ms 2024-09-17T10:51:55.936 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:55.936 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:55.936 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:55 smithi006 ceph-mon[63965]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T10:51:55.936 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T10:51:55.936 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:55.936 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:55 smithi006 ceph-mon[63965]: Reconfiguring daemon osd.6 on smithi149 2024-09-17T10:51:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:55 smithi149 ceph-mon[49697]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T10:51:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T10:51:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:56.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:55 smithi149 ceph-mon[49697]: Reconfiguring daemon osd.6 on smithi149 2024-09-17T10:51:56.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 152455 10.95 MB/sec execute 228 sec latency 33.589 ms 2024-09-17T10:51:56.950 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:56 smithi006 ceph-mon[63965]: pgmap v31: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 668 op/s 2024-09-17T10:51:56.950 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:56.950 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:56.950 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T10:51:56.950 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:57.080 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:56 smithi149 ceph-mon[49697]: pgmap v31: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 668 op/s 2024-09-17T10:51:57.080 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:57.080 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:57.080 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T10:51:57.080 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:57.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 152786 10.95 MB/sec execute 229 sec latency 61.865 ms 2024-09-17T10:51:57.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:57 smithi149 ceph-mon[49697]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T10:51:57.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:57 smithi149 ceph-mon[49697]: Reconfiguring daemon osd.7 on smithi149 2024-09-17T10:51:57.969 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:57 smithi006 ceph-mon[63965]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T10:51:57.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:57 smithi006 ceph-mon[63965]: Reconfiguring daemon osd.7 on smithi149 2024-09-17T10:51:58.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 153049 10.93 MB/sec execute 230 sec latency 59.079 ms 2024-09-17T10:51:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:58 smithi149 ceph-mon[49697]: pgmap v32: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.2 MiB/s wr, 673 op/s 2024-09-17T10:51:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:58 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:58 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:58 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.hmuwiq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T10:51:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:58 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:58.978 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:58 smithi006 ceph-mon[63965]: pgmap v32: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.2 MiB/s wr, 673 op/s 2024-09-17T10:51:58.979 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:58 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:58.979 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:58 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:51:58.979 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:58 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.hmuwiq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T10:51:58.979 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:58 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:51:59.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 153270 10.89 MB/sec execute 231 sec latency 59.185 ms 2024-09-17T10:52:00.019 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:59 smithi006 ceph-mon[63965]: Reconfiguring mds.foofs.smithi149.hmuwiq (monmap changed)... 2024-09-17T10:52:00.019 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:59 smithi006 ceph-mon[63965]: Reconfiguring daemon mds.foofs.smithi149.hmuwiq on smithi149 2024-09-17T10:52:00.019 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:59 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:00.019 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:51:59 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:59 smithi149 ceph-mon[49697]: Reconfiguring mds.foofs.smithi149.hmuwiq (monmap changed)... 2024-09-17T10:52:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:59 smithi149 ceph-mon[49697]: Reconfiguring daemon mds.foofs.smithi149.hmuwiq on smithi149 2024-09-17T10:52:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:59 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:51:59 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:00.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 153688 10.88 MB/sec execute 232 sec latency 46.673 ms 2024-09-17T10:52:01.028 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: pgmap v33: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.6 MiB/s wr, 556 op/s 2024-09-17T10:52:01.028 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:52:01.028 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:01.028 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:01.028 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:01.028 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:52:01.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi006"} : dispatch 2024-09-17T10:52:01.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi006"}]': finished 2024-09-17T10:52:01.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi149"} : dispatch 2024-09-17T10:52:01.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi149"}]': finished 2024-09-17T10:52:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: pgmap v33: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.6 MiB/s wr, 556 op/s 2024-09-17T10:52:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:52:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:01.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:52:01.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi006"} : dispatch 2024-09-17T10:52:01.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi006"}]': finished 2024-09-17T10:52:01.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi149"} : dispatch 2024-09-17T10:52:01.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi149"}]': finished 2024-09-17T10:52:01.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 154026 10.87 MB/sec execute 233 sec latency 68.920 ms 2024-09-17T10:52:02.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:01 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all mon 2024-09-17T10:52:02.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:01 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all mon 2024-09-17T10:52:02.813 INFO:teuthology.orchestra.run.smithi006.stdout: 5 154334 10.84 MB/sec execute 234 sec latency 58.212 ms 2024-09-17T10:52:03.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:02 smithi006 ceph-mon[63965]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 776 op/s 2024-09-17T10:52:03.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:03.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T10:52:03.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:52:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:02 smithi149 ceph-mon[49697]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 776 op/s 2024-09-17T10:52:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T10:52:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:52:03.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 154694 10.84 MB/sec execute 235 sec latency 48.435 ms 2024-09-17T10:52:04.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:03 smithi006 ceph-mon[63965]: Upgrade: Updating crash.smithi006 (1/2) 2024-09-17T10:52:04.053 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:03 smithi006 ceph-mon[63965]: Deploying daemon crash.smithi006 on smithi006 2024-09-17T10:52:04.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:03 smithi149 ceph-mon[49697]: Upgrade: Updating crash.smithi006 (1/2) 2024-09-17T10:52:04.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:03 smithi149 ceph-mon[49697]: Deploying daemon crash.smithi006 on smithi006 2024-09-17T10:52:04.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 155160 10.84 MB/sec execute 236 sec latency 62.366 ms 2024-09-17T10:52:05.074 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:04 smithi006 ceph-mon[63965]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-17T10:52:05.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:04 smithi149 ceph-mon[49697]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-17T10:52:05.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 155768 10.90 MB/sec execute 237 sec latency 26.736 ms 2024-09-17T10:52:06.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 156299 10.93 MB/sec execute 238 sec latency 31.509 ms 2024-09-17T10:52:07.018 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:06 smithi006 ceph-mon[63965]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 672 op/s 2024-09-17T10:52:07.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:06 smithi149 ceph-mon[49697]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 672 op/s 2024-09-17T10:52:07.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 156669 10.92 MB/sec execute 239 sec latency 41.924 ms 2024-09-17T10:52:08.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 156934 10.90 MB/sec execute 240 sec latency 59.394 ms 2024-09-17T10:52:09.088 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:08 smithi006 ceph-mon[63965]: pgmap v37: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.5 MiB/s wr, 558 op/s 2024-09-17T10:52:09.089 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:09.089 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:08 smithi149 ceph-mon[49697]: pgmap v37: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.5 MiB/s wr, 558 op/s 2024-09-17T10:52:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:08 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:08 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:09.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 157265 10.87 MB/sec execute 241 sec latency 83.517 ms 2024-09-17T10:52:10.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:10 smithi006 ceph-mon[63965]: Upgrade: Updating crash.smithi149 (2/2) 2024-09-17T10:52:10.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:10.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T10:52:10.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:52:10.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:10 smithi006 ceph-mon[63965]: Deploying daemon crash.smithi149 on smithi149 2024-09-17T10:52:10.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:10 smithi006 ceph-mon[63965]: pgmap v38: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 454 op/s 2024-09-17T10:52:10.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:10 smithi149 ceph-mon[49697]: Upgrade: Updating crash.smithi149 (2/2) 2024-09-17T10:52:10.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:10 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:10.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:10 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T10:52:10.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:10 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:52:10.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:10 smithi149 ceph-mon[49697]: Deploying daemon crash.smithi149 on smithi149 2024-09-17T10:52:10.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:10 smithi149 ceph-mon[49697]: pgmap v38: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 454 op/s 2024-09-17T10:52:10.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 157806 10.87 MB/sec execute 242 sec latency 39.554 ms 2024-09-17T10:52:11.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 158301 10.88 MB/sec execute 243 sec latency 44.165 ms 2024-09-17T10:52:12.639 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:12 smithi006 ceph-mon[63965]: pgmap v39: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 710 op/s 2024-09-17T10:52:12.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 158566 10.85 MB/sec execute 244 sec latency 122.186 ms 2024-09-17T10:52:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:12 smithi149 ceph-mon[49697]: pgmap v39: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 710 op/s 2024-09-17T10:52:13.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 158839 10.86 MB/sec execute 245 sec latency 57.105 ms 2024-09-17T10:52:14.436 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:14.436 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:14.436 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:52:14.436 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:14 smithi006 ceph-mon[63965]: pgmap v40: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.8 MiB/s wr, 491 op/s 2024-09-17T10:52:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:52:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:14 smithi149 ceph-mon[49697]: pgmap v40: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.8 MiB/s wr, 491 op/s 2024-09-17T10:52:14.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 159161 10.86 MB/sec execute 246 sec latency 44.367 ms 2024-09-17T10:52:15.109 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:52:15.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:52:15.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:52:15.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 159496 10.88 MB/sec execute 247 sec latency 50.635 ms 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (3m) 67s ago 12m 18.5M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 starting - - - - 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 starting - - - - 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (3m) 67s ago 12m 42.1M - dad864ee21e9 89df198e1a9b 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (6m) 67s ago 6m 37.4M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (6m) 54s ago 6m 14.5M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (2m) 67s ago 13m 488M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (103s) 54s ago 12m 452M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:52:16.180 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (84s) 67s ago 13m 32.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (60s) 54s ago 11m 23.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (6m) 67s ago 6m 276M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (6m) 54s ago 6m 85.9M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (3m) 67s ago 12m 9675k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (2m) 54s ago 11m 12.4M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (11m) 67s ago 11m 480M 1937M 17.2.0 e1d6a67b021e 0b397addaf64 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (10m) 67s ago 10m 399M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (10m) 67s ago 10m 309M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (9m) 67s ago 9m 443M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (9m) 54s ago 9m 473M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (9m) 54s ago 9m 538M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (8m) 54s ago 8m 458M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:52:16.181 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (8m) 54s ago 8m 328M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:52:16.182 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (109s) 67s ago 11m 44.1M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:52:16.470 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:16 smithi006 ceph-mon[63965]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:16.471 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:16 smithi006 ceph-mon[63965]: pgmap v41: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 600 op/s 2024-09-17T10:52:16.471 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:16 smithi006 ceph-mon[63965]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:16.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:16 smithi149 ceph-mon[49697]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:16.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:16 smithi149 ceph-mon[49697]: pgmap v41: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 600 op/s 2024-09-17T10:52:16.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:16 smithi149 ceph-mon[49697]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:52:16.725 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:52:16.726 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:52:16.726 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:52:16.726 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T10:52:16.726 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T10:52:16.726 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:52:16.726 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:52:16.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 159768 10.86 MB/sec execute 248 sec latency 67.217 ms 2024-09-17T10:52:17.179 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "4/23 daemons upgraded", 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading crash daemons", 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:52:17.180 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:52:17.499 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:17 smithi006 ceph-mon[63965]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:17.500 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:17 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2131124262' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:17.500 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:17 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:17.500 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:17 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:17 smithi149 ceph-mon[49697]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:17 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2131124262' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:17 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:17.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:17 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:17.711 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:52:17.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 160018 10.85 MB/sec execute 249 sec latency 47.610 ms 2024-09-17T10:52:18.543 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:18 smithi006 ceph-mon[63965]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:18.543 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:18 smithi006 ceph-mon[63965]: pgmap v42: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.3 MiB/s wr, 575 op/s 2024-09-17T10:52:18.544 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:18 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3660211528' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:52:18.544 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:18 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:18.544 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:18 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:18.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:18 smithi149 ceph-mon[49697]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:18.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:18 smithi149 ceph-mon[49697]: pgmap v42: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.3 MiB/s wr, 575 op/s 2024-09-17T10:52:18.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:18 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3660211528' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:52:18.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:18 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:18.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:18 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:18.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 160229 10.82 MB/sec execute 250 sec latency 63.034 ms 2024-09-17T10:52:19.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 160581 10.81 MB/sec execute 251 sec latency 35.917 ms 2024-09-17T10:52:20.573 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:20 smithi006 ceph-mon[63965]: pgmap v43: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-17T10:52:20.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:20 smithi149 ceph-mon[49697]: pgmap v43: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-17T10:52:20.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 161153 10.81 MB/sec execute 252 sec latency 34.436 ms 2024-09-17T10:52:21.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 161781 10.83 MB/sec execute 253 sec latency 30.858 ms 2024-09-17T10:52:22.585 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:22 smithi006 ceph-mon[63965]: pgmap v44: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.7 MiB/s wr, 663 op/s 2024-09-17T10:52:22.585 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:22.585 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:22.585 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:22.585 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:22.814 INFO:teuthology.orchestra.run.smithi006.stdout: 5 162533 10.87 MB/sec execute 254 sec latency 30.055 ms 2024-09-17T10:52:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:22 smithi149 ceph-mon[49697]: pgmap v44: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.7 MiB/s wr, 663 op/s 2024-09-17T10:52:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:23.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 163138 10.92 MB/sec execute 255 sec latency 35.833 ms 2024-09-17T10:52:24.349 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:24.349 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:24.349 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:24 smithi006 ceph-mon[63965]: pgmap v45: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 406 op/s 2024-09-17T10:52:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:24 smithi149 ceph-mon[49697]: pgmap v45: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 406 op/s 2024-09-17T10:52:24.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 163694 10.93 MB/sec execute 256 sec latency 36.733 ms 2024-09-17T10:52:25.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 164002 10.92 MB/sec execute 257 sec latency 66.536 ms 2024-09-17T10:52:26.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:26 smithi006 ceph-mon[63965]: pgmap v46: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.2 MiB/s wr, 565 op/s 2024-09-17T10:52:26.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 164295 10.89 MB/sec execute 258 sec latency 79.744 ms 2024-09-17T10:52:26.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:26 smithi149 ceph-mon[49697]: pgmap v46: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.2 MiB/s wr, 565 op/s 2024-09-17T10:52:27.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 164605 10.88 MB/sec execute 259 sec latency 64.375 ms 2024-09-17T10:52:28.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:28 smithi006 ceph-mon[63965]: pgmap v47: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.3 MiB/s wr, 647 op/s 2024-09-17T10:52:28.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:28.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:28.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:52:28.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:52:28.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:28.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:28.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 164928 10.85 MB/sec execute 260 sec latency 68.193 ms 2024-09-17T10:52:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:28 smithi149 ceph-mon[49697]: pgmap v47: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.3 MiB/s wr, 647 op/s 2024-09-17T10:52:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:52:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:52:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:28.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi006"} : dispatch 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi006"}]': finished 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi149"} : dispatch 2024-09-17T10:52:29.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi149"}]': finished 2024-09-17T10:52:29.724 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-17T10:52:29.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 165296 10.86 MB/sec execute 261 sec latency 41.188 ms 2024-09-17T10:52:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:52:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi006"} : dispatch 2024-09-17T10:52:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi006"}]': finished 2024-09-17T10:52:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi149"} : dispatch 2024-09-17T10:52:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi149"}]': finished 2024-09-17T10:52:29.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-17T10:52:30.642 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:30 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all crash 2024-09-17T10:52:30.642 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:30 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T10:52:30.642 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:30 smithi006 ceph-mon[63965]: Upgrade: osd.0 is safe to restart 2024-09-17T10:52:30.642 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:30 smithi006 ceph-mon[63965]: pgmap v48: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.8 MiB/s wr, 550 op/s 2024-09-17T10:52:30.642 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:52:30.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 165644 10.85 MB/sec execute 262 sec latency 56.367 ms 2024-09-17T10:52:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:30 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all crash 2024-09-17T10:52:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:30 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T10:52:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:30 smithi149 ceph-mon[49697]: Upgrade: osd.0 is safe to restart 2024-09-17T10:52:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:30 smithi149 ceph-mon[49697]: pgmap v48: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.8 MiB/s wr, 550 op/s 2024-09-17T10:52:30.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:52:31.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 166009 10.86 MB/sec execute 263 sec latency 38.071 ms 2024-09-17T10:52:32.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:32 smithi006 ceph-mon[63965]: Upgrade: Updating osd.0 2024-09-17T10:52:32.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:32.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T10:52:32.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:52:32.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:32 smithi006 ceph-mon[63965]: Deploying daemon osd.0 on smithi006 2024-09-17T10:52:32.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:32 smithi006 ceph-mon[63965]: pgmap v49: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 749 op/s 2024-09-17T10:52:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:32 smithi149 ceph-mon[49697]: Upgrade: Updating osd.0 2024-09-17T10:52:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T10:52:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:52:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:32 smithi149 ceph-mon[49697]: Deploying daemon osd.0 on smithi006 2024-09-17T10:52:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:32 smithi149 ceph-mon[49697]: pgmap v49: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 749 op/s 2024-09-17T10:52:32.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 166296 10.85 MB/sec execute 264 sec latency 50.683 ms 2024-09-17T10:52:33.294 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:33 smithi006 ceph-mon[63965]: mgrmap e43: smithi006.plhvdm(active, since 92s), standbys: smithi149.ijckag 2024-09-17T10:52:33.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:33 smithi149 ceph-mon[49697]: mgrmap e43: smithi006.plhvdm(active, since 92s), standbys: smithi149.ijckag 2024-09-17T10:52:33.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 166636 10.87 MB/sec execute 265 sec latency 56.506 ms 2024-09-17T10:52:34.320 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:34 smithi006 ceph-mon[63965]: pgmap v50: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 549 op/s 2024-09-17T10:52:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:34 smithi149 ceph-mon[49697]: pgmap v50: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 549 op/s 2024-09-17T10:52:34.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 166877 10.84 MB/sec execute 266 sec latency 54.478 ms 2024-09-17T10:52:35.559 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:35 smithi006 systemd[1]: Stopping Ceph osd.0 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:52:35.809 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:35 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[34367]: 2024-09-17T10:52:35.688+0000 7fdd0c851700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:52:35.810 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:35 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[34367]: 2024-09-17T10:52:35.688+0000 7fdd0c851700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-17T10:52:35.810 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:35 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[34367]: 2024-09-17T10:52:35.688+0000 7fdd0c851700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T10:52:35.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167087 10.83 MB/sec execute 267 sec latency 55.207 ms 2024-09-17T10:52:36.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:36 smithi006 ceph-mon[63965]: pgmap v51: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.6 MiB/s wr, 638 op/s 2024-09-17T10:52:36.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:36 smithi006 ceph-mon[63965]: osd.0 marked itself down and dead 2024-09-17T10:52:36.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:36 smithi149 ceph-mon[49697]: pgmap v51: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.6 MiB/s wr, 638 op/s 2024-09-17T10:52:36.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:36 smithi149 ceph-mon[49697]: osd.0 marked itself down and dead 2024-09-17T10:52:36.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167225 10.80 MB/sec execute 268 sec latency 293.081 ms 2024-09-17T10:52:37.485 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:37 smithi006 podman[68285]: 2024-09-17 10:52:37.226572961 +0000 UTC m=+1.701931050 container died 0b397addaf64f286d350d6bae670d543827cbb7664440e716584011a593ce6ef (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, ceph=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vendor=Red Hat, Inc.) 2024-09-17T10:52:37.737 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:37 smithi006 ceph-mon[63965]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:52:37.738 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:37 smithi006 ceph-mon[63965]: osdmap e73: 8 total, 7 up, 8 in 2024-09-17T10:52:37.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167270 10.76 MB/sec execute 269 sec latency 1103.789 ms 2024-09-17T10:52:37.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:37 smithi149 ceph-mon[49697]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:52:37.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:37 smithi149 ceph-mon[49697]: osdmap e73: 8 total, 7 up, 8 in 2024-09-17T10:52:37.999 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:37 smithi006 podman[68285]: 2024-09-17 10:52:37.801254475 +0000 UTC m=+2.276612566 container cleanup 0b397addaf64f286d350d6bae670d543827cbb7664440e716584011a593ce6ef (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, distribution-scope=public, RELEASE=HEAD, name=centos-stream, version=8, ceph=True, build-date=2022-05-03T08:36:31.336870) 2024-09-17T10:52:37.999 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:37 smithi006 bash[68285]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0 2024-09-17T10:52:38.262 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:38 smithi006 podman[68300]: 2024-09-17 10:52:38.145515588 +0000 UTC m=+0.922223700 container remove 0b397addaf64f286d350d6bae670d543827cbb7664440e716584011a593ce6ef (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., architecture=x86_64, maintainer=Guillaume Abrioux , vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_CLEAN=True) 2024-09-17T10:52:38.765 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:38 smithi006 ceph-mon[63965]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.5 MiB/s wr, 447 op/s 2024-09-17T10:52:38.765 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:38 smithi006 ceph-mon[63965]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T10:52:38.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167534 10.75 MB/sec execute 270 sec latency 38.392 ms 2024-09-17T10:52:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:38 smithi149 ceph-mon[49697]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.5 MiB/s wr, 447 op/s 2024-09-17T10:52:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:38 smithi149 ceph-mon[49697]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T10:52:39.023 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:38 smithi006 podman[68381]: 2024-09-17 10:52:38.681424326 +0000 UTC m=+0.022348515 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:52:39.023 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:38 smithi006 podman[68381]: 2024-09-17 10:52:38.933755373 +0000 UTC m=+0.274679543 container create 69b63ede56f0a0ad67c0fdb77054053486607ea19300c0dcd2a0bf39e448a776 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-17T10:52:39.791 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:39 smithi006 podman[68381]: 2024-09-17 10:52:39.552986218 +0000 UTC m=+0.893910398 container init 69b63ede56f0a0ad67c0fdb77054053486607ea19300c0dcd2a0bf39e448a776 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493) 2024-09-17T10:52:39.791 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:39 smithi006 podman[68381]: 2024-09-17 10:52:39.568398763 +0000 UTC m=+0.909322937 container start 69b63ede56f0a0ad67c0fdb77054053486607ea19300c0dcd2a0bf39e448a776 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T10:52:39.791 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:39 smithi006 podman[68381]: 2024-09-17 10:52:39.722269193 +0000 UTC m=+1.063193365 container attach 69b63ede56f0a0ad67c0fdb77054053486607ea19300c0dcd2a0bf39e448a776 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T10:52:39.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167807 10.72 MB/sec execute 271 sec latency 53.396 ms 2024-09-17T10:52:40.052 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:39 smithi006 podman[68381]: 2024-09-17 10:52:39.931362344 +0000 UTC m=+1.272286532 container died 69b63ede56f0a0ad67c0fdb77054053486607ea19300c0dcd2a0bf39e448a776 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-17T10:52:40.555 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:40 smithi006 ceph-mon[63965]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.3 MiB/s wr, 260 op/s 2024-09-17T10:52:40.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 168150 10.71 MB/sec execute 272 sec latency 53.388 ms 2024-09-17T10:52:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:40 smithi149 ceph-mon[49697]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.3 MiB/s wr, 260 op/s 2024-09-17T10:52:41.059 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:41 smithi006 podman[68403]: 2024-09-17 10:52:41.053421457 +0000 UTC m=+1.113581465 container remove 69b63ede56f0a0ad67c0fdb77054053486607ea19300c0dcd2a0bf39e448a776 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-17T10:52:41.339 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:41 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-17T10:52:41.339 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:41 smithi006 systemd[1]: Stopped Ceph osd.0 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:52:41.339 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:41 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 53.463s CPU time. 2024-09-17T10:52:41.589 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:41 smithi006 systemd[1]: Starting Ceph osd.0 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:52:41.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 168440 10.68 MB/sec execute 273 sec latency 60.715 ms 2024-09-17T10:52:41.841 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:41 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 210/1599 objects degraded (13.133%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T10:52:41.842 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:41 smithi006 podman[68497]: 2024-09-17 10:52:41.593921066 +0000 UTC m=+0.025448690 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:52:41.842 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:41 smithi006 podman[68497]: 2024-09-17 10:52:41.721405662 +0000 UTC m=+0.152933268 container create 534b7e067cf42199d102ea46815c3dcc540d73d0a371f24b802a5152e12ef837 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate, RELEASE=main-7498493, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-17T10:52:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:41 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 210/1599 objects degraded (13.133%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T10:52:42.093 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:42 smithi006 podman[68497]: 2024-09-17 10:52:42.024783863 +0000 UTC m=+0.456311481 container init 534b7e067cf42199d102ea46815c3dcc540d73d0a371f24b802a5152e12ef837 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-7498493, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-17T10:52:42.093 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:42 smithi006 podman[68497]: 2024-09-17 10:52:42.038834845 +0000 UTC m=+0.470362456 container start 534b7e067cf42199d102ea46815c3dcc540d73d0a371f24b802a5152e12ef837 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-7498493, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T10:52:42.363 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:42 smithi006 podman[68497]: 2024-09-17 10:52:42.159275936 +0000 UTC m=+0.590803553 container attach 534b7e067cf42199d102ea46815c3dcc540d73d0a371f24b802a5152e12ef837 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-17T10:52:42.363 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:42 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:52:42.363 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:42 smithi006 bash[68497]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:52:42.363 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:42 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:52:42.364 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:42 smithi006 bash[68497]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:52:42.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:42 smithi006 ceph-mon[63965]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 468 op/s; 210/1599 objects degraded (13.133%) 2024-09-17T10:52:42.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 168711 10.67 MB/sec execute 274 sec latency 65.634 ms 2024-09-17T10:52:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:42 smithi149 ceph-mon[49697]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 468 op/s; 210/1599 objects degraded (13.133%) 2024-09-17T10:52:43.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 168992 10.67 MB/sec execute 275 sec latency 62.068 ms 2024-09-17T10:52:43.900 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:43 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T10:52:43.900 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:43 smithi006 bash[68497]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T10:52:43.900 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:43 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T10:52:43.900 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:43 smithi006 bash[68497]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T10:52:44.154 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T10:52:44.154 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 bash[68497]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T10:52:44.154 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T10:52:44.155 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 bash[68497]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T10:52:44.155 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T10:52:44.155 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 bash[68497]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T10:52:44.155 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T10:52:44.155 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 bash[68497]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T10:52:44.155 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate[68512]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T10:52:44.156 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 bash[68497]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T10:52:44.156 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 podman[68497]: 2024-09-17 10:52:44.12456507 +0000 UTC m=+2.556092698 container died 534b7e067cf42199d102ea46815c3dcc540d73d0a371f24b802a5152e12ef837 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:52:44.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:44 smithi006 ceph-mon[63965]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.2 MiB/s wr, 335 op/s; 210/1599 objects degraded (13.133%) 2024-09-17T10:52:44.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 169325 10.66 MB/sec execute 276 sec latency 81.918 ms 2024-09-17T10:52:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:44 smithi149 ceph-mon[49697]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.2 MiB/s wr, 335 op/s; 210/1599 objects degraded (13.133%) 2024-09-17T10:52:45.190 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:44 smithi006 podman[68663]: 2024-09-17 10:52:44.993143999 +0000 UTC m=+0.861128838 container remove 534b7e067cf42199d102ea46815c3dcc540d73d0a371f24b802a5152e12ef837 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T10:52:45.453 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:45 smithi006 podman[68719]: 2024-09-17 10:52:45.13552724 +0000 UTC m=+0.019297970 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:52:45.453 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:45 smithi006 podman[68719]: 2024-09-17 10:52:45.259837541 +0000 UTC m=+0.143608251 container create 2775f74b7e4dc7970ec76caaf2d2d1ef7e3beb5953a944f72803ebe1676b9b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-17T10:52:45.702 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:45 smithi006 podman[68719]: 2024-09-17 10:52:45.579582239 +0000 UTC m=+0.463352966 container init 2775f74b7e4dc7970ec76caaf2d2d1ef7e3beb5953a944f72803ebe1676b9b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-17T10:52:45.702 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:45 smithi006 podman[68719]: 2024-09-17 10:52:45.589039273 +0000 UTC m=+0.472809993 container start 2775f74b7e4dc7970ec76caaf2d2d1ef7e3beb5953a944f72803ebe1676b9b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-7498493) 2024-09-17T10:52:45.702 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:45 smithi006 ceph-osd[68736]: -- 172.21.15.6:0/3454862593 <== mon.0 v2:172.21.15.6:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aef0b841e0 con 0x55aef0b14000 2024-09-17T10:52:45.702 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:45 smithi006 bash[68719]: 2775f74b7e4dc7970ec76caaf2d2d1ef7e3beb5953a944f72803ebe1676b9b61 2024-09-17T10:52:45.702 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:45 smithi006 systemd[1]: Started Ceph osd.0 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:52:45.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 169627 10.66 MB/sec execute 277 sec latency 72.244 ms 2024-09-17T10:52:46.211 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:46.211 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:52:46.211 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:46 smithi006 ceph-mon[63965]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.1 MiB/s wr, 211 op/s; 218/1611 objects degraded (13.532%) 2024-09-17T10:52:46.211 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:46.211 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:46.211 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:52:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:52:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:46 smithi149 ceph-mon[49697]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.1 MiB/s wr, 211 op/s; 218/1611 objects degraded (13.532%) 2024-09-17T10:52:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:46.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:46.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:52:46.815 INFO:teuthology.orchestra.run.smithi006.stdout: 5 170018 10.67 MB/sec execute 278 sec latency 29.113 ms 2024-09-17T10:52:47.479 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:47 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 218/1611 objects degraded (13.532%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T10:52:47.479 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:47 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[68732]: 2024-09-17T10:52:47.462+0000 7fb7495c4640 -1 Falling back to public interface 2024-09-17T10:52:47.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:47 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 218/1611 objects degraded (13.532%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T10:52:47.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 170442 10.68 MB/sec execute 279 sec latency 53.840 ms 2024-09-17T10:52:48.318 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:52:48.490 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:48 smithi006 ceph-mon[63965]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 377 op/s; 215/1593 objects degraded (13.497%) 2024-09-17T10:52:48.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:48 smithi149 ceph-mon[49697]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 377 op/s; 215/1593 objects degraded (13.497%) 2024-09-17T10:52:48.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 170734 10.68 MB/sec execute 280 sec latency 51.180 ms 2024-09-17T10:52:49.485 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (4m) 27s ago 13m 18.6M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (42s) 27s ago 13m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (36s) 32s ago 12m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (3m) 27s ago 12m 42.4M - dad864ee21e9 89df198e1a9b 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (7m) 27s ago 7m 37.8M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (7m) 32s ago 7m 14.7M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (3m) 27s ago 14m 509M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (2m) 32s ago 12m 453M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (118s) 27s ago 14m 44.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:52:49.486 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (93s) 32s ago 12m 35.2M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (6m) 27s ago 6m 280M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (6m) 32s ago 6m 86.0M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (3m) 27s ago 12m 9202k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (3m) 32s ago 12m 11.5M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 starting - - - 1937M 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (11m) 27s ago 11m 477M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (10m) 27s ago 10m 397M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (10m) 27s ago 10m 513M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (9m) 32s ago 9m 513M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:52:49.487 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (9m) 32s ago 9m 334M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:52:49.488 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (9m) 32s ago 9m 530M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:52:49.488 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (8m) 32s ago 8m 391M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:52:49.488 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (2m) 27s ago 12m 48.6M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:52:49.552 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:49 smithi006 ceph-mon[63965]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:49.553 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:49 smithi006 ceph-mon[63965]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:49.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:49 smithi149 ceph-mon[49697]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:49.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:49 smithi149 ceph-mon[49697]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:49.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 170919 10.65 MB/sec execute 281 sec latency 43.323 ms 2024-09-17T10:52:50.122 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:52:50.123 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:52:50.124 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:52:50.124 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:52:50.124 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T10:52:50.124 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T10:52:50.124 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:52:50.124 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:52:50.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:50 smithi006 ceph-mon[63965]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:50.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:50 smithi006 ceph-mon[63965]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.8 MiB/s wr, 316 op/s; 215/1593 objects degraded (13.497%) 2024-09-17T10:52:50.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:50 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/728952679' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:50.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:50 smithi149 ceph-mon[49697]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:50.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:50 smithi149 ceph-mon[49697]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.8 MiB/s wr, 316 op/s; 215/1593 objects degraded (13.497%) 2024-09-17T10:52:50.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:50 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/728952679' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:52:50.709 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "6/23 daemons upgraded", 2024-09-17T10:52:50.710 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:52:50.710 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:52:50.710 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:52:50.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 171136 10.63 MB/sec execute 282 sec latency 63.791 ms 2024-09-17T10:52:51.348 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:51 smithi006 ceph-mon[63965]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:51.385 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 215/1593 objects degraded (13.497%), 29 pgs degraded 2024-09-17T10:52:51.385 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T10:52:51.385 INFO:teuthology.orchestra.run.smithi006.stdout: osd.0 (root=default,host=smithi006) is down 2024-09-17T10:52:51.385 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 215/1593 objects degraded (13.497%), 29 pgs degraded 2024-09-17T10:52:51.385 INFO:teuthology.orchestra.run.smithi006.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-17T10:52:51.385 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.0 is active+undersized+degraded, acting [2,1] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-17T10:52:51.386 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-17T10:52:51.387 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-17T10:52:51.388 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-17T10:52:51.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:51 smithi149 ceph-mon[49697]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:52:51.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 171389 10.61 MB/sec execute 283 sec latency 60.438 ms 2024-09-17T10:52:52.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:52 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/524013907' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:52:52.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:52 smithi006 ceph-mon[63965]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 509 op/s; 228/1668 objects degraded (13.669%) 2024-09-17T10:52:52.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:52 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 215/1593 objects degraded (13.497%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T10:52:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:52 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/524013907' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:52:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:52 smithi149 ceph-mon[49697]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 509 op/s; 228/1668 objects degraded (13.669%) 2024-09-17T10:52:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:52 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 215/1593 objects degraded (13.497%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T10:52:52.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 171757 10.60 MB/sec execute 284 sec latency 46.616 ms 2024-09-17T10:52:53.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 172069 10.58 MB/sec execute 285 sec latency 38.746 ms 2024-09-17T10:52:54.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 172394 10.57 MB/sec execute 286 sec latency 58.574 ms 2024-09-17T10:52:54.915 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:54 smithi006 ceph-mon[63965]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 370 op/s; 228/1668 objects degraded (13.669%) 2024-09-17T10:52:54.915 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:54.915 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:54.915 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:54.915 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:54 smithi149 ceph-mon[49697]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 370 op/s; 228/1668 objects degraded (13.669%) 2024-09-17T10:52:55.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:55.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:55.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:55.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:52:55.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 172674 10.57 MB/sec execute 287 sec latency 61.402 ms 2024-09-17T10:52:55.967 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:55 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[68732]: 2024-09-17T10:52:55.809+0000 7fb7495c4640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-17T10:52:56.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 173054 10.57 MB/sec execute 288 sec latency 50.986 ms 2024-09-17T10:52:56.992 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:56 smithi006 ceph-mon[63965]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 403 op/s; 218/1611 objects degraded (13.532%) 2024-09-17T10:52:56.992 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:56 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[68732]: 2024-09-17T10:52:56.927+0000 7fb7495c4640 -1 osd.0 72 log_to_monitors true 2024-09-17T10:52:57.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:56 smithi149 ceph-mon[49697]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 403 op/s; 218/1611 objects degraded (13.532%) 2024-09-17T10:52:57.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 173653 10.61 MB/sec execute 289 sec latency 35.629 ms 2024-09-17T10:52:57.999 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:57 smithi006 ceph-mon[63965]: from='osd.0 [v2:172.21.15.6:6802/3031547706,v1:172.21.15.6:6803/3031547706]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-17T10:52:57.999 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:57 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 218/1611 objects degraded (13.532%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T10:52:57.999 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 10:52:57 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[68732]: 2024-09-17T10:52:57.957+0000 7fb74136e640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:52:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:57 smithi149 ceph-mon[49697]: from='osd.0 [v2:172.21.15.6:6802/3031547706,v1:172.21.15.6:6803/3031547706]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-17T10:52:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:57 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 218/1611 objects degraded (13.532%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T10:52:58.816 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174133 10.62 MB/sec execute 290 sec latency 33.769 ms 2024-09-17T10:52:59.015 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:58 smithi006 ceph-mon[63965]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 539 op/s; 205/1554 objects degraded (13.192%) 2024-09-17T10:52:59.016 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:58 smithi006 ceph-mon[63965]: from='osd.0 [v2:172.21.15.6:6802/3031547706,v1:172.21.15.6:6803/3031547706]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T10:52:59.016 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:58 smithi006 ceph-mon[63965]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T10:52:59.016 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:58 smithi006 ceph-mon[63965]: from='osd.0 [v2:172.21.15.6:6802/3031547706,v1:172.21.15.6:6803/3031547706]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]} : dispatch 2024-09-17T10:52:59.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:58 smithi149 ceph-mon[49697]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 539 op/s; 205/1554 objects degraded (13.192%) 2024-09-17T10:52:59.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:58 smithi149 ceph-mon[49697]: from='osd.0 [v2:172.21.15.6:6802/3031547706,v1:172.21.15.6:6803/3031547706]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T10:52:59.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:58 smithi149 ceph-mon[49697]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T10:52:59.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:58 smithi149 ceph-mon[49697]: from='osd.0 [v2:172.21.15.6:6802/3031547706,v1:172.21.15.6:6803/3031547706]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]} : dispatch 2024-09-17T10:52:59.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174324 10.59 MB/sec execute 291 sec latency 698.766 ms 2024-09-17T10:53:00.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:59 smithi006 ceph-mon[63965]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:53:00.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:59 smithi006 ceph-mon[63965]: osd.0 [v2:172.21.15.6:6802/3031547706,v1:172.21.15.6:6803/3031547706] boot 2024-09-17T10:53:00.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:59 smithi006 ceph-mon[63965]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T10:53:00.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:59 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T10:53:00.165 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:52:59 smithi006 ceph-mon[63965]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T10:53:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:59 smithi149 ceph-mon[49697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:53:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:59 smithi149 ceph-mon[49697]: osd.0 [v2:172.21.15.6:6802/3031547706,v1:172.21.15.6:6803/3031547706] boot 2024-09-17T10:53:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:59 smithi149 ceph-mon[49697]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T10:53:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:59 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T10:53:00.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:52:59 smithi149 ceph-mon[49697]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T10:53:00.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174387 10.57 MB/sec execute 292 sec latency 1698.795 ms 2024-09-17T10:53:01.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:00 smithi006 ceph-mon[63965]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.2 MiB/s wr, 280 op/s; 205/1554 objects degraded (13.192%) 2024-09-17T10:53:01.176 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:01.176 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:53:01.176 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:00 smithi006 ceph-mon[63965]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T10:53:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:00 smithi149 ceph-mon[49697]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.2 MiB/s wr, 280 op/s; 205/1554 objects degraded (13.192%) 2024-09-17T10:53:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:53:01.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:00 smithi149 ceph-mon[49697]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T10:53:01.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174552 10.55 MB/sec execute 293 sec latency 1972.068 ms 2024-09-17T10:53:02.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:02.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:02.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:53:02.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:53:02.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:02.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:02.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: pgmap v70: 97 pgs: 14 remapped+peering, 5 active+undersized, 3 active+undersized+degraded, 75 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 7.2 MiB/s wr, 609 op/s; 19/1794 objects degraded (1.059%); 22 B/s, 4 objects/s recovering 2024-09-17T10:53:02.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:53:02.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:02.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:02.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:02.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T10:53:02.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T10:53:02.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:02 smithi006 ceph-mon[63965]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T10:53:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:53:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:53:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: pgmap v70: 97 pgs: 14 remapped+peering, 5 active+undersized, 3 active+undersized+degraded, 75 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 7.2 MiB/s wr, 609 op/s; 19/1794 objects degraded (1.059%); 22 B/s, 4 objects/s recovering 2024-09-17T10:53:02.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:53:02.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:02.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:02.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:02.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T10:53:02.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T10:53:02.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:02 smithi149 ceph-mon[49697]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T10:53:02.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174833 10.54 MB/sec execute 294 sec latency 643.429 ms 2024-09-17T10:53:03.410 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:03 smithi006 ceph-mon[63965]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T10:53:03.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:03 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 19/1794 objects degraded (1.059%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:03 smithi149 ceph-mon[49697]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T10:53:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:03 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 19/1794 objects degraded (1.059%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:03.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 175055 10.51 MB/sec execute 295 sec latency 1000.322 ms 2024-09-17T10:53:04.176 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:04 smithi006 ceph-mon[63965]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T10:53:04.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:04 smithi006 ceph-mon[63965]: pgmap v73: 97 pgs: 1 peering, 13 remapped+peering, 83 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 7.3 MiB/s wr, 612 op/s; 22 B/s, 9 objects/s recovering 2024-09-17T10:53:04.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:04 smithi006 ceph-mon[63965]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T10:53:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:04 smithi149 ceph-mon[49697]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T10:53:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:04 smithi149 ceph-mon[49697]: pgmap v73: 97 pgs: 1 peering, 13 remapped+peering, 83 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 7.3 MiB/s wr, 612 op/s; 22 B/s, 9 objects/s recovering 2024-09-17T10:53:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:04 smithi149 ceph-mon[49697]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T10:53:04.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 175516 10.51 MB/sec execute 296 sec latency 888.999 ms 2024-09-17T10:53:05.370 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:05 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 19/1794 objects degraded (1.059%), 3 pgs degraded) 2024-09-17T10:53:05.370 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:05 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:53:05.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:05 smithi006 ceph-mon[63965]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T10:53:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:05 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 19/1794 objects degraded (1.059%), 3 pgs degraded) 2024-09-17T10:53:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:05 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:53:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:05 smithi149 ceph-mon[49697]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T10:53:05.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 175808 10.49 MB/sec execute 297 sec latency 801.897 ms 2024-09-17T10:53:06.182 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:06 smithi006 ceph-mon[63965]: pgmap v76: 97 pgs: 1 peering, 11 remapped+peering, 85 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s wr, 201 op/s; 0 B/s, 18 objects/s recovering 2024-09-17T10:53:06.182 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:06 smithi006 ceph-mon[63965]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T10:53:06.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:06 smithi149 ceph-mon[49697]: pgmap v76: 97 pgs: 1 peering, 11 remapped+peering, 85 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s wr, 201 op/s; 0 B/s, 18 objects/s recovering 2024-09-17T10:53:06.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:06 smithi149 ceph-mon[49697]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T10:53:06.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 176279 10.49 MB/sec execute 298 sec latency 970.466 ms 2024-09-17T10:53:07.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 176889 10.54 MB/sec execute 299 sec latency 26.518 ms 2024-09-17T10:53:08.719 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:08 smithi006 ceph-mon[63965]: pgmap v78: 97 pgs: 3 peering, 4 remapped+peering, 90 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 3.3 MiB/s wr, 441 op/s; 3.7 MiB/s, 30 objects/s recovering 2024-09-17T10:53:08.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 177421 10.57 MB/sec execute 300 sec latency 29.054 ms 2024-09-17T10:53:08.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:08 smithi149 ceph-mon[49697]: pgmap v78: 97 pgs: 3 peering, 4 remapped+peering, 90 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 3.3 MiB/s wr, 441 op/s; 3.7 MiB/s, 30 objects/s recovering 2024-09-17T10:53:09.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 177807 10.57 MB/sec execute 301 sec latency 37.240 ms 2024-09-17T10:53:10.633 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:10 smithi006 ceph-mon[63965]: pgmap v79: 97 pgs: 2 peering, 95 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.4 MiB/s wr, 365 op/s; 2.7 MiB/s, 45 objects/s recovering 2024-09-17T10:53:10.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 178135 10.55 MB/sec execute 302 sec latency 49.404 ms 2024-09-17T10:53:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:10 smithi149 ceph-mon[49697]: pgmap v79: 97 pgs: 2 peering, 95 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.4 MiB/s wr, 365 op/s; 2.7 MiB/s, 45 objects/s recovering 2024-09-17T10:53:11.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 178438 10.53 MB/sec execute 303 sec latency 88.024 ms 2024-09-17T10:53:12.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:12 smithi006 ceph-mon[63965]: pgmap v80: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 728 op/s; 2.2 MiB/s, 37 objects/s recovering 2024-09-17T10:53:12.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 178763 10.52 MB/sec execute 304 sec latency 69.451 ms 2024-09-17T10:53:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:12 smithi149 ceph-mon[49697]: pgmap v80: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 728 op/s; 2.2 MiB/s, 37 objects/s recovering 2024-09-17T10:53:13.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 179079 10.51 MB/sec execute 305 sec latency 95.052 ms 2024-09-17T10:53:14.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:14 smithi006 ceph-mon[63965]: pgmap v81: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 661 op/s; 1.9 MiB/s, 32 objects/s recovering 2024-09-17T10:53:14.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 179727 10.53 MB/sec execute 306 sec latency 54.600 ms 2024-09-17T10:53:14.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:14 smithi149 ceph-mon[49697]: pgmap v81: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 661 op/s; 1.9 MiB/s, 32 objects/s recovering 2024-09-17T10:53:15.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 180208 10.55 MB/sec execute 307 sec latency 55.220 ms 2024-09-17T10:53:16.260 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:16.261 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:53:16.261 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:16 smithi006 ceph-mon[63965]: pgmap v82: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 675 op/s; 1.6 MiB/s, 21 objects/s recovering 2024-09-17T10:53:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:53:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:16 smithi149 ceph-mon[49697]: pgmap v82: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 675 op/s; 1.6 MiB/s, 21 objects/s recovering 2024-09-17T10:53:16.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 180637 10.56 MB/sec execute 308 sec latency 76.970 ms 2024-09-17T10:53:17.057 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:17 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T10:53:17.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:17 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T10:53:17.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:17 smithi006 ceph-mon[63965]: Upgrade: osd.1 is safe to restart 2024-09-17T10:53:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:17 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T10:53:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:17 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T10:53:17.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:17 smithi149 ceph-mon[49697]: Upgrade: osd.1 is safe to restart 2024-09-17T10:53:17.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 181006 10.57 MB/sec execute 309 sec latency 42.878 ms 2024-09-17T10:53:18.067 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:18 smithi006 ceph-mon[63965]: pgmap v83: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 559 op/s; 0 B/s, 12 objects/s recovering 2024-09-17T10:53:18.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:18 smithi149 ceph-mon[49697]: pgmap v83: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 559 op/s; 0 B/s, 12 objects/s recovering 2024-09-17T10:53:18.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 181214 10.56 MB/sec execute 310 sec latency 56.121 ms 2024-09-17T10:53:19.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 181412 10.54 MB/sec execute 311 sec latency 49.468 ms 2024-09-17T10:53:20.115 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:19 smithi006 ceph-mon[63965]: Upgrade: Updating osd.1 2024-09-17T10:53:20.116 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:20.116 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T10:53:20.116 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:53:20.116 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:19 smithi006 ceph-mon[63965]: Deploying daemon osd.1 on smithi006 2024-09-17T10:53:20.116 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:19 smithi006 ceph-mon[63965]: pgmap v84: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.8 MiB/s wr, 547 op/s; 0 B/s, 11 objects/s recovering 2024-09-17T10:53:20.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:19 smithi149 ceph-mon[49697]: Upgrade: Updating osd.1 2024-09-17T10:53:20.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:20.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T10:53:20.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:53:20.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:19 smithi149 ceph-mon[49697]: Deploying daemon osd.1 on smithi006 2024-09-17T10:53:20.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:19 smithi149 ceph-mon[49697]: pgmap v84: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.8 MiB/s wr, 547 op/s; 0 B/s, 11 objects/s recovering 2024-09-17T10:53:20.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 181712 10.52 MB/sec execute 312 sec latency 65.984 ms 2024-09-17T10:53:21.134 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:21 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:21.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:21 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:21.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 182289 10.53 MB/sec execute 313 sec latency 30.606 ms 2024-09-17T10:53:22.022 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:53:22.140 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:22 smithi006 ceph-mon[63965]: pgmap v85: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.2 MiB/s wr, 701 op/s 2024-09-17T10:53:22.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:22 smithi149 ceph-mon[49697]: pgmap v85: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.2 MiB/s wr, 701 op/s 2024-09-17T10:53:22.817 INFO:teuthology.orchestra.run.smithi006.stdout: 5 182665 10.52 MB/sec execute 314 sec latency 40.072 ms 2024-09-17T10:53:23.036 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (4m) 29s ago 13m 17.5M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (75s) 29s ago 13m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (70s) 65s ago 13m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (4m) 29s ago 13m 42.2M - dad864ee21e9 89df198e1a9b 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (7m) 29s ago 7m 37.9M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (7m) 65s ago 7m 14.7M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (3m) 29s ago 14m 511M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (2m) 65s ago 13m 453M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (2m) 29s ago 15m 47.7M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (2m) 65s ago 13m 35.2M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (7m) 29s ago 7m 287M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:53:23.037 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (7m) 65s ago 7m 86.0M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (4m) 29s ago 13m 9248k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (3m) 65s ago 12m 11.5M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (37s) 29s ago 12m 12.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (11m) 29s ago 11m 520M 1937M 17.2.0 e1d6a67b021e 382ff0928a7a 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (11m) 29s ago 11m 434M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (10m) 29s ago 10m 524M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (10m) 65s ago 10m 513M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (10m) 65s ago 10m 334M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (9m) 65s ago 9m 530M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (9m) 65s ago 9m 391M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:53:23.038 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (2m) 29s ago 12m 48.8M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:53:23.167 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:23 smithi006 ceph-mon[63965]: from='client.44163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:23.167 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:23 smithi006 ceph-mon[63965]: from='client.44167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:23.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:23 smithi149 ceph-mon[49697]: from='client.44163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:23.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:23 smithi149 ceph-mon[49697]: from='client.44167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:53:23.792 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:53:23.793 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:53:23.793 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:53:23.793 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:53:23.793 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:53:23.793 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T10:53:23.793 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 5 2024-09-17T10:53:23.793 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:53:23.793 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:53:23.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 183042 10.51 MB/sec execute 315 sec latency 56.250 ms 2024-09-17T10:53:24.194 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:24 smithi006 ceph-mon[63965]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:24.194 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:24 smithi006 ceph-mon[63965]: pgmap v86: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 451 op/s 2024-09-17T10:53:24.194 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:24 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1298518339' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:24.425 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:53:24.425 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:53:24.425 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:53:24.425 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:53:24.425 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:53:24.425 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:53:24.425 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:53:24.425 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:53:24.426 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:53:24.426 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "7/23 daemons upgraded", 2024-09-17T10:53:24.426 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:53:24.426 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:53:24.426 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:53:24.446 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:24 smithi006 systemd[1]: Stopping Ceph osd.1 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:53:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:24 smithi149 ceph-mon[49697]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:24 smithi149 ceph-mon[49697]: pgmap v86: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 451 op/s 2024-09-17T10:53:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:24 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/1298518339' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:24.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 183300 10.50 MB/sec execute 316 sec latency 64.309 ms 2024-09-17T10:53:24.824 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:24 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[38294]: 2024-09-17T10:53:24.742+0000 7f0068f99700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:53:24.824 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:24 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[38294]: 2024-09-17T10:53:24.742+0000 7f0068f99700 -1 osd.1 83 *** Got signal Terminated *** 2024-09-17T10:53:24.824 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:24 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[38294]: 2024-09-17T10:53:24.742+0000 7f0068f99700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T10:53:25.158 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down 2024-09-17T10:53:25.159 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T10:53:25.159 INFO:teuthology.orchestra.run.smithi006.stdout: osd.1 (root=default,host=smithi006) is down 2024-09-17T10:53:25.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:25 smithi006 ceph-mon[63965]: from='client.44175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:25.226 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:25 smithi006 ceph-mon[63965]: osd.1 marked itself down and dead 2024-09-17T10:53:25.357 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:25 smithi149 ceph-mon[49697]: from='client.44175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:25.357 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:25 smithi149 ceph-mon[49697]: osd.1 marked itself down and dead 2024-09-17T10:53:25.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 183449 10.47 MB/sec execute 317 sec latency 729.246 ms 2024-09-17T10:53:25.951 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:25 smithi006 podman[70708]: 2024-09-17 10:53:25.687880104 +0000 UTC m=+1.098450821 container died 382ff0928a7acf1248fe251495dc9e7db1ec28b8901cfcb1b7fb32aeed35ff2b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, release=754, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, architecture=x86_64, RELEASE=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T10:53:26.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:26 smithi006 ceph-mon[63965]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:53:26.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:26 smithi006 ceph-mon[63965]: osdmap e84: 8 total, 7 up, 8 in 2024-09-17T10:53:26.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:26 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3682297053' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:53:26.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:26 smithi006 ceph-mon[63965]: pgmap v88: 97 pgs: 13 peering, 8 stale+active+clean, 76 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.8 MiB/s wr, 478 op/s 2024-09-17T10:53:26.283 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:26 smithi006 podman[70708]: 2024-09-17 10:53:26.212712941 +0000 UTC m=+1.623283656 container cleanup 382ff0928a7acf1248fe251495dc9e7db1ec28b8901cfcb1b7fb32aeed35ff2b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, release=754, vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, version=8, name=centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , architecture=x86_64, distribution-scope=public, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8) 2024-09-17T10:53:26.283 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:26 smithi006 bash[70708]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1 2024-09-17T10:53:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:26 smithi149 ceph-mon[49697]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:53:26.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:26 smithi149 ceph-mon[49697]: osdmap e84: 8 total, 7 up, 8 in 2024-09-17T10:53:26.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:26 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3682297053' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:53:26.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:26 smithi149 ceph-mon[49697]: pgmap v88: 97 pgs: 13 peering, 8 stale+active+clean, 76 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.8 MiB/s wr, 478 op/s 2024-09-17T10:53:26.534 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:26 smithi006 podman[70746]: 2024-09-17 10:53:26.532254209 +0000 UTC m=+0.843272437 container remove 382ff0928a7acf1248fe251495dc9e7db1ec28b8901cfcb1b7fb32aeed35ff2b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, release=754, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream) 2024-09-17T10:53:26.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 183760 10.48 MB/sec execute 318 sec latency 992.920 ms 2024-09-17T10:53:27.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:27 smithi006 ceph-mon[63965]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T10:53:27.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:27 smithi006 ceph-mon[63965]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T10:53:27.296 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:27 smithi006 podman[70823]: 2024-09-17 10:53:27.06685031 +0000 UTC m=+0.028724801 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:53:27.296 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:27 smithi006 podman[70823]: 2024-09-17 10:53:27.250430339 +0000 UTC m=+0.212304819 container create d0f47322c7de335ac2e7952da67572d57913da83aac9f84179e1002b0320695f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T10:53:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:27 smithi149 ceph-mon[49697]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T10:53:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:27 smithi149 ceph-mon[49697]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T10:53:27.804 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:27 smithi006 podman[70823]: 2024-09-17 10:53:27.696235423 +0000 UTC m=+0.658109919 container init d0f47322c7de335ac2e7952da67572d57913da83aac9f84179e1002b0320695f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T10:53:27.804 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:27 smithi006 podman[70823]: 2024-09-17 10:53:27.702483972 +0000 UTC m=+0.664358446 container start d0f47322c7de335ac2e7952da67572d57913da83aac9f84179e1002b0320695f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-7498493) 2024-09-17T10:53:27.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 184065 10.48 MB/sec execute 319 sec latency 43.742 ms 2024-09-17T10:53:28.055 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:28 smithi006 ceph-mon[63965]: pgmap v90: 97 pgs: 28 peering, 69 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 610 op/s 2024-09-17T10:53:28.055 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:27 smithi006 podman[70823]: 2024-09-17 10:53:27.804610615 +0000 UTC m=+0.766485098 container attach d0f47322c7de335ac2e7952da67572d57913da83aac9f84179e1002b0320695f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True) 2024-09-17T10:53:28.306 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:28 smithi006 podman[70846]: 2024-09-17 10:53:28.116368583 +0000 UTC m=+0.038107190 container died d0f47322c7de335ac2e7952da67572d57913da83aac9f84179e1002b0320695f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T10:53:28.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:28 smithi149 ceph-mon[49697]: pgmap v90: 97 pgs: 28 peering, 69 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 610 op/s 2024-09-17T10:53:28.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 184717 10.52 MB/sec execute 320 sec latency 32.367 ms 2024-09-17T10:53:29.312 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:29 smithi006 podman[70846]: 2024-09-17 10:53:29.063238026 +0000 UTC m=+0.984976625 container remove d0f47322c7de335ac2e7952da67572d57913da83aac9f84179e1002b0320695f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-7498493, io.buildah.version=1.37.2) 2024-09-17T10:53:29.312 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:29 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-17T10:53:29.312 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:29 smithi006 systemd[1]: Stopped Ceph osd.1 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:53:29.312 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:29 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.1.service: Consumed 3min 21.549s CPU time. 2024-09-17T10:53:29.563 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:29 smithi006 systemd[1]: Starting Ceph osd.1 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:53:29.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:29 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 20/1605 objects degraded (1.246%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:29.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 185210 10.52 MB/sec execute 321 sec latency 36.956 ms 2024-09-17T10:53:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:29 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 20/1605 objects degraded (1.246%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:30.065 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:29 smithi006 podman[70941]: 2024-09-17 10:53:29.732058226 +0000 UTC m=+0.030393700 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:53:30.065 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:29 smithi006 podman[70941]: 2024-09-17 10:53:29.865231934 +0000 UTC m=+0.163567388 container create 351ddafbf395723db3a78d4165a68731fe899469370c8b1c1c3af094eae670f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-17T10:53:30.316 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:30 smithi006 podman[70941]: 2024-09-17 10:53:30.270798611 +0000 UTC m=+0.569134086 container init 351ddafbf395723db3a78d4165a68731fe899469370c8b1c1c3af094eae670f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-17T10:53:30.316 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:30 smithi006 podman[70941]: 2024-09-17 10:53:30.277555062 +0000 UTC m=+0.575890525 container start 351ddafbf395723db3a78d4165a68731fe899469370c8b1c1c3af094eae670f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T10:53:30.567 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:30 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:53:30.567 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:30 smithi006 podman[70941]: 2024-09-17 10:53:30.453999382 +0000 UTC m=+0.752334858 container attach 351ddafbf395723db3a78d4165a68731fe899469370c8b1c1c3af094eae670f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T10:53:30.567 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:30 smithi006 bash[70941]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:53:30.567 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:30 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:53:30.567 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:30 smithi006 bash[70941]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:53:30.817 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:30 smithi006 ceph-mon[63965]: pgmap v91: 97 pgs: 2 active+undersized, 28 peering, 2 active+undersized+degraded, 65 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 364 op/s; 20/1605 objects degraded (1.246%) 2024-09-17T10:53:30.818 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:30.818 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:53:30.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 185591 10.51 MB/sec execute 322 sec latency 46.781 ms 2024-09-17T10:53:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:30 smithi149 ceph-mon[49697]: pgmap v91: 97 pgs: 2 active+undersized, 28 peering, 2 active+undersized+degraded, 65 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 364 op/s; 20/1605 objects degraded (1.246%) 2024-09-17T10:53:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:30.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:53:31.818 INFO:teuthology.orchestra.run.smithi006.stdout: 5 185982 10.51 MB/sec execute 323 sec latency 56.560 ms 2024-09-17T10:53:31.822 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:31 smithi006 ceph-mon[63965]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-17T10:53:31.822 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:31 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T10:53:31.822 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:31 smithi006 bash[70941]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T10:53:31.823 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:31 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T10:53:31.823 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:31 smithi006 bash[70941]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T10:53:31.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:31 smithi149 ceph-mon[49697]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-17T10:53:32.073 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T10:53:32.074 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 bash[70941]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T10:53:32.074 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T10:53:32.074 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 bash[70941]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T10:53:32.334 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 bash[70941]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T10:53:32.334 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T10:53:32.334 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T10:53:32.334 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 bash[70941]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T10:53:32.334 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate[70954]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T10:53:32.334 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 bash[70941]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T10:53:32.334 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:32 smithi006 podman[70941]: 2024-09-17 10:53:32.1121636 +0000 UTC m=+2.410499067 container died 351ddafbf395723db3a78d4165a68731fe899469370c8b1c1c3af094eae670f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-7498493, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:53:32.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 186437 10.49 MB/sec execute 324 sec latency 36.910 ms 2024-09-17T10:53:32.840 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:32 smithi006 ceph-mon[63965]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.5 MiB/s wr, 686 op/s; 212/1815 objects degraded (11.680%) 2024-09-17T10:53:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:32 smithi149 ceph-mon[49697]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.5 MiB/s wr, 686 op/s; 212/1815 objects degraded (11.680%) 2024-09-17T10:53:33.597 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:33 smithi006 podman[71111]: 2024-09-17 10:53:33.352278801 +0000 UTC m=+1.230794515 container remove 351ddafbf395723db3a78d4165a68731fe899469370c8b1c1c3af094eae670f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T10:53:33.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 186888 10.50 MB/sec execute 325 sec latency 50.788 ms 2024-09-17T10:53:33.826 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:33 smithi006 podman[71164]: 2024-09-17 10:53:33.558502106 +0000 UTC m=+0.025110950 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:53:33.826 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:33 smithi006 podman[71164]: 2024-09-17 10:53:33.720795536 +0000 UTC m=+0.187404368 container create 35586f41af8fcaaf0b2a06ae61022d3f279ce06e2ecb9a8963d9926060c02814 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-17T10:53:34.376 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:34 smithi006 podman[71164]: 2024-09-17 10:53:34.116954048 +0000 UTC m=+0.583562880 container init 35586f41af8fcaaf0b2a06ae61022d3f279ce06e2ecb9a8963d9926060c02814 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T10:53:34.376 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:34 smithi006 podman[71164]: 2024-09-17 10:53:34.127688698 +0000 UTC m=+0.594297525 container start 35586f41af8fcaaf0b2a06ae61022d3f279ce06e2ecb9a8963d9926060c02814 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T10:53:34.376 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:34 smithi006 ceph-osd[71181]: -- 172.21.15.6:0/528279192 <== mon.0 v2:172.21.15.6:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fbc1dee1e0 con 0x55fbc0f87c00 2024-09-17T10:53:34.376 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:34 smithi006 bash[71164]: 35586f41af8fcaaf0b2a06ae61022d3f279ce06e2ecb9a8963d9926060c02814 2024-09-17T10:53:34.376 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:34 smithi006 systemd[1]: Started Ceph osd.1 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:53:34.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 187310 10.52 MB/sec execute 326 sec latency 38.129 ms 2024-09-17T10:53:34.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:34 smithi006 ceph-mon[63965]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.7 MiB/s wr, 677 op/s; 211/1815 objects degraded (11.625%) 2024-09-17T10:53:34.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:34.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:34.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:53:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:34 smithi149 ceph-mon[49697]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.7 MiB/s wr, 677 op/s; 211/1815 objects degraded (11.625%) 2024-09-17T10:53:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:53:35.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 187634 10.52 MB/sec execute 327 sec latency 51.024 ms 2024-09-17T10:53:35.905 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:35 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 201/1794 objects degraded (11.204%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:35.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:35 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 201/1794 objects degraded (11.204%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:36.440 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:36 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[71176]: 2024-09-17T10:53:36.246+0000 7eff2d1a6640 -1 Falling back to public interface 2024-09-17T10:53:36.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 187949 10.53 MB/sec execute 328 sec latency 47.607 ms 2024-09-17T10:53:36.954 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:36 smithi006 ceph-mon[63965]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.8 MiB/s wr, 527 op/s; 201/1794 objects degraded (11.204%) 2024-09-17T10:53:36.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:36 smithi149 ceph-mon[49697]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.8 MiB/s wr, 527 op/s; 201/1794 objects degraded (11.204%) 2024-09-17T10:53:37.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 188205 10.52 MB/sec execute 329 sec latency 70.276 ms 2024-09-17T10:53:38.764 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:38 smithi006 ceph-mon[63965]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 516 op/s; 171/1683 objects degraded (10.160%) 2024-09-17T10:53:38.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 188395 10.51 MB/sec execute 330 sec latency 66.259 ms 2024-09-17T10:53:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:38 smithi149 ceph-mon[49697]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 516 op/s; 171/1683 objects degraded (10.160%) 2024-09-17T10:53:39.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 188592 10.49 MB/sec execute 331 sec latency 60.297 ms 2024-09-17T10:53:40.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:40 smithi006 ceph-mon[63965]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.5 MiB/s wr, 505 op/s; 168/1689 objects degraded (9.947%) 2024-09-17T10:53:40.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 188841 10.47 MB/sec execute 332 sec latency 60.424 ms 2024-09-17T10:53:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:40 smithi149 ceph-mon[49697]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.5 MiB/s wr, 505 op/s; 168/1689 objects degraded (9.947%) 2024-09-17T10:53:41.810 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:41 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 183/1737 objects degraded (10.535%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:41.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 189049 10.44 MB/sec execute 333 sec latency 63.670 ms 2024-09-17T10:53:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:41 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 183/1737 objects degraded (10.535%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:42.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:42 smithi006 ceph-mon[63965]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 634 op/s; 183/1737 objects degraded (10.535%) 2024-09-17T10:53:42.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:42.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:42.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 189298 10.42 MB/sec execute 334 sec latency 50.688 ms 2024-09-17T10:53:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:42 smithi149 ceph-mon[49697]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 634 op/s; 183/1737 objects degraded (10.535%) 2024-09-17T10:53:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:43.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 189657 10.43 MB/sec execute 335 sec latency 37.510 ms 2024-09-17T10:53:44.799 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:44 smithi006 ceph-mon[63965]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.5 MiB/s wr, 418 op/s; 182/1707 objects degraded (10.662%) 2024-09-17T10:53:44.799 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:44 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:44.799 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:44 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:44.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 190166 10.43 MB/sec execute 336 sec latency 38.414 ms 2024-09-17T10:53:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:44 smithi149 ceph-mon[49697]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.5 MiB/s wr, 418 op/s; 182/1707 objects degraded (10.662%) 2024-09-17T10:53:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:44 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:44 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:45.553 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:45 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[71176]: 2024-09-17T10:53:45.520+0000 7eff2d1a6640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-17T10:53:45.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 190553 10.42 MB/sec execute 337 sec latency 37.056 ms 2024-09-17T10:53:46.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:46.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:53:46.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:45 smithi006 ceph-mon[63965]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.5 MiB/s wr, 429 op/s; 179/1650 objects degraded (10.848%) 2024-09-17T10:53:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:53:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:45 smithi149 ceph-mon[49697]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.5 MiB/s wr, 429 op/s; 179/1650 objects degraded (10.848%) 2024-09-17T10:53:46.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 190936 10.44 MB/sec execute 338 sec latency 46.028 ms 2024-09-17T10:53:46.831 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:46 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[71176]: 2024-09-17T10:53:46.714+0000 7eff2d1a6640 -1 osd.1 83 log_to_monitors true 2024-09-17T10:53:47.088 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:47 smithi006 ceph-mon[63965]: from='osd.1 [v2:172.21.15.6:6810/2234106147,v1:172.21.15.6:6811/2234106147]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-17T10:53:47.088 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 10:53:47 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[71176]: 2024-09-17T10:53:47.070+0000 7eff24f50640 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:53:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:47 smithi149 ceph-mon[49697]: from='osd.1 [v2:172.21.15.6:6810/2234106147,v1:172.21.15.6:6811/2234106147]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-17T10:53:47.819 INFO:teuthology.orchestra.run.smithi006.stdout: 5 191235 10.43 MB/sec execute 339 sec latency 82.146 ms 2024-09-17T10:53:48.104 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:48 smithi006 ceph-mon[63965]: from='osd.1 [v2:172.21.15.6:6810/2234106147,v1:172.21.15.6:6811/2234106147]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T10:53:48.104 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:48 smithi006 ceph-mon[63965]: osdmap e86: 8 total, 7 up, 8 in 2024-09-17T10:53:48.104 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:48 smithi006 ceph-mon[63965]: from='osd.1 [v2:172.21.15.6:6810/2234106147,v1:172.21.15.6:6811/2234106147]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]} : dispatch 2024-09-17T10:53:48.104 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:48 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 179/1650 objects degraded (10.848%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:48.104 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:48 smithi006 ceph-mon[63965]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 429 op/s; 179/1620 objects degraded (11.049%) 2024-09-17T10:53:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:48 smithi149 ceph-mon[49697]: from='osd.1 [v2:172.21.15.6:6810/2234106147,v1:172.21.15.6:6811/2234106147]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T10:53:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:48 smithi149 ceph-mon[49697]: osdmap e86: 8 total, 7 up, 8 in 2024-09-17T10:53:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:48 smithi149 ceph-mon[49697]: from='osd.1 [v2:172.21.15.6:6810/2234106147,v1:172.21.15.6:6811/2234106147]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]} : dispatch 2024-09-17T10:53:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:48 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 179/1650 objects degraded (10.848%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:48 smithi149 ceph-mon[49697]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 429 op/s; 179/1620 objects degraded (11.049%) 2024-09-17T10:53:48.820 INFO:teuthology.orchestra.run.smithi006.stdout: 5 191423 10.42 MB/sec execute 340 sec latency 753.467 ms 2024-09-17T10:53:49.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:49 smithi006 ceph-mon[63965]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:53:49.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:49 smithi006 ceph-mon[63965]: osd.1 [v2:172.21.15.6:6810/2234106147,v1:172.21.15.6:6811/2234106147] boot 2024-09-17T10:53:49.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:49 smithi006 ceph-mon[63965]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T10:53:49.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:49 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T10:53:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:49 smithi149 ceph-mon[49697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:53:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:49 smithi149 ceph-mon[49697]: osd.1 [v2:172.21.15.6:6810/2234106147,v1:172.21.15.6:6811/2234106147] boot 2024-09-17T10:53:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:49 smithi149 ceph-mon[49697]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T10:53:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:49 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T10:53:49.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 191423 10.39 MB/sec execute 341 sec latency 1758.281 ms 2024-09-17T10:53:50.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T10:53:50.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:50.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:50.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:53:50.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:53:50.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:50.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:50.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.5 MiB/s wr, 415 op/s; 182/1650 objects degraded (11.030%) 2024-09-17T10:53:50.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:53:50.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:50.174 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:50.174 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:50.174 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T10:53:50.174 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T10:53:50.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-17T10:53:50.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:50 smithi006 ceph-mon[63965]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T10:53:50.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.5 MiB/s wr, 415 op/s; 182/1650 objects degraded (11.030%) 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T10:53:50.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T10:53:50.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-17T10:53:50.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:50 smithi149 ceph-mon[49697]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T10:53:50.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 191641 10.39 MB/sec execute 342 sec latency 2102.960 ms 2024-09-17T10:53:51.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 191864 10.37 MB/sec execute 343 sec latency 533.188 ms 2024-09-17T10:53:52.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:52 smithi006 ceph-mon[63965]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T10:53:52.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:52 smithi006 ceph-mon[63965]: pgmap v107: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized, 7 active+undersized+degraded, 73 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 406 op/s; 175/1635 objects degraded (10.703%); 2.0 MiB/s, 26 objects/s recovering 2024-09-17T10:53:52.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:52 smithi149 ceph-mon[49697]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T10:53:52.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:52 smithi149 ceph-mon[49697]: pgmap v107: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized, 7 active+undersized+degraded, 73 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 406 op/s; 175/1635 objects degraded (10.703%); 2.0 MiB/s, 26 objects/s recovering 2024-09-17T10:53:52.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 192207 10.36 MB/sec execute 344 sec latency 799.707 ms 2024-09-17T10:53:53.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:53 smithi006 ceph-mon[63965]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T10:53:53.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:53 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 175/1635 objects degraded (10.703%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:53.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:53 smithi149 ceph-mon[49697]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T10:53:53.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:53 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 175/1635 objects degraded (10.703%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:53.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 192479 10.34 MB/sec execute 345 sec latency 967.464 ms 2024-09-17T10:53:54.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:54 smithi006 ceph-mon[63965]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T10:53:54.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:54 smithi006 ceph-mon[63965]: pgmap v110: 97 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 378 op/s; 135/1629 objects degraded (8.287%); 2.1 MiB/s, 67 keys/s, 27 objects/s recovering 2024-09-17T10:53:54.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:54 smithi006 ceph-mon[63965]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T10:53:54.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:54 smithi149 ceph-mon[49697]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T10:53:54.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:54 smithi149 ceph-mon[49697]: pgmap v110: 97 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 378 op/s; 135/1629 objects degraded (8.287%); 2.1 MiB/s, 67 keys/s, 27 objects/s recovering 2024-09-17T10:53:54.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:54 smithi149 ceph-mon[49697]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T10:53:54.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 192808 10.34 MB/sec execute 346 sec latency 1011.541 ms 2024-09-17T10:53:55.808 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:53:55.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 193373 10.36 MB/sec execute 347 sec latency 487.070 ms 2024-09-17T10:53:56.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:56 smithi006 ceph-mon[63965]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T10:53:56.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:56 smithi006 ceph-mon[63965]: pgmap v113: 97 pgs: 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 90 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s wr, 67 op/s; 77/1644 objects degraded (4.684%); 113 KiB/s, 67 keys/s, 20 objects/s recovering 2024-09-17T10:53:56.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:56 smithi006 ceph-mon[63965]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:56.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:56 smithi149 ceph-mon[49697]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T10:53:56.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:56 smithi149 ceph-mon[49697]: pgmap v113: 97 pgs: 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 90 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s wr, 67 op/s; 77/1644 objects degraded (4.684%); 113 KiB/s, 67 keys/s, 20 objects/s recovering 2024-09-17T10:53:56.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:56 smithi149 ceph-mon[49697]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:56.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 193880 10.36 MB/sec execute 348 sec latency 27.814 ms 2024-09-17T10:53:56.996 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:53:56.996 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (5m) 14s ago 14m 17.7M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:53:56.996 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (109s) 14s ago 14m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:53:56.996 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (104s) 99s ago 13m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (5m) 14s ago 14m 42.7M - dad864ee21e9 89df198e1a9b 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (8m) 14s ago 8m 37.9M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (8m) 99s ago 8m 14.7M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (4m) 14s ago 15m 515M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (3m) 99s ago 13m 453M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (3m) 14s ago 15m 49.5M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (2m) 99s ago 13m 35.2M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (8m) 14s ago 8m 283M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (7m) 99s ago 7m 86.0M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (4m) 14s ago 14m 9059k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (4m) 99s ago 13m 11.5M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (71s) 14s ago 12m 240M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (23s) 14s ago 12m 12.2M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:53:56.997 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (11m) 14s ago 11m 512M 1937M 17.2.0 e1d6a67b021e 60ca0fdb4a51 2024-09-17T10:53:56.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (11m) 14s ago 11m 594M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:53:56.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (11m) 99s ago 11m 513M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:53:56.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (10m) 99s ago 10m 334M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:53:56.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (10m) 99s ago 10m 530M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:53:56.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (9m) 99s ago 9m 391M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:53:56.998 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (3m) 14s ago 13m 48.9M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:53:57.115 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:57 smithi006 ceph-mon[63965]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:57.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:57 smithi149 ceph-mon[49697]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:53:57.734 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 6 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:53:57.735 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:53:57.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 194305 10.38 MB/sec execute 349 sec latency 29.270 ms 2024-09-17T10:53:58.130 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:58 smithi006 ceph-mon[63965]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:58.131 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:58 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 77/1644 objects degraded (4.684%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:58.131 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:58 smithi006 ceph-mon[63965]: pgmap v114: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 590 op/s; 84 KiB/s, 37 objects/s recovering 2024-09-17T10:53:58.131 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:58 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1460308104' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:58.328 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:53:58.328 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:53:58.328 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:53:58.328 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:53:58.328 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:53:58.328 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:53:58.328 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:53:58.328 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:53:58.329 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:53:58.329 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "8/23 daemons upgraded", 2024-09-17T10:53:58.329 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:53:58.329 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:53:58.329 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:53:58.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:58 smithi149 ceph-mon[49697]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:58.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:58 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 77/1644 objects degraded (4.684%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T10:53:58.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:58 smithi149 ceph-mon[49697]: pgmap v114: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 590 op/s; 84 KiB/s, 37 objects/s recovering 2024-09-17T10:53:58.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:58 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/1460308104' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:53:58.824 INFO:teuthology.orchestra.run.smithi006.stdout: 5 194755 10.39 MB/sec execute 350 sec latency 47.631 ms 2024-09-17T10:53:59.114 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:53:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:59 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 77/1644 objects degraded (4.684%), 3 pgs degraded) 2024-09-17T10:53:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:59 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:53:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:53:59 smithi006 ceph-mon[63965]: from='client.44199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:59 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 77/1644 objects degraded (4.684%), 3 pgs degraded) 2024-09-17T10:53:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:59 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:53:59.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:53:59 smithi149 ceph-mon[49697]: from='client.44199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:53:59.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 195139 10.39 MB/sec execute 351 sec latency 45.853 ms 2024-09-17T10:54:00.167 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:00 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2138663769' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:54:00.168 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:00 smithi006 ceph-mon[63965]: pgmap v115: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 555 op/s; 0 B/s, 27 objects/s recovering 2024-09-17T10:54:00.168 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:00.168 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:54:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:00 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2138663769' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:54:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:00 smithi149 ceph-mon[49697]: pgmap v115: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 555 op/s; 0 B/s, 27 objects/s recovering 2024-09-17T10:54:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:54:00.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 195459 10.38 MB/sec execute 352 sec latency 64.189 ms 2024-09-17T10:54:01.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 195874 10.37 MB/sec execute 353 sec latency 40.565 ms 2024-09-17T10:54:02.748 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:02 smithi006 ceph-mon[63965]: pgmap v116: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 562 op/s; 0 B/s, 21 objects/s recovering 2024-09-17T10:54:02.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 196185 10.36 MB/sec execute 354 sec latency 40.949 ms 2024-09-17T10:54:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:02 smithi149 ceph-mon[49697]: pgmap v116: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 562 op/s; 0 B/s, 21 objects/s recovering 2024-09-17T10:54:03.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 196467 10.36 MB/sec execute 355 sec latency 47.558 ms 2024-09-17T10:54:04.808 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:04 smithi006 ceph-mon[63965]: pgmap v117: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.3 MiB/s wr, 611 op/s; 0 B/s, 18 objects/s recovering 2024-09-17T10:54:04.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 196716 10.36 MB/sec execute 356 sec latency 90.482 ms 2024-09-17T10:54:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:04 smithi149 ceph-mon[49697]: pgmap v117: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.3 MiB/s wr, 611 op/s; 0 B/s, 18 objects/s recovering 2024-09-17T10:54:05.560 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:05 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T10:54:05.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:05 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T10:54:05.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:05 smithi006 ceph-mon[63965]: Upgrade: osd.2 is safe to restart 2024-09-17T10:54:05.561 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:05 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:05.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 197064 10.34 MB/sec execute 357 sec latency 48.335 ms 2024-09-17T10:54:05.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:05 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T10:54:05.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:05 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T10:54:05.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:05 smithi149 ceph-mon[49697]: Upgrade: osd.2 is safe to restart 2024-09-17T10:54:05.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:05 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:06.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:06 smithi006 ceph-mon[63965]: pgmap v118: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 569 op/s; 0 B/s, 16 objects/s recovering 2024-09-17T10:54:06.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 197297 10.33 MB/sec execute 358 sec latency 77.694 ms 2024-09-17T10:54:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:06 smithi149 ceph-mon[49697]: pgmap v118: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 569 op/s; 0 B/s, 16 objects/s recovering 2024-09-17T10:54:07.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 197457 10.31 MB/sec execute 359 sec latency 68.820 ms 2024-09-17T10:54:08.392 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:08 smithi006 ceph-mon[63965]: Upgrade: Updating osd.2 2024-09-17T10:54:08.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:08.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T10:54:08.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:54:08.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:08 smithi006 ceph-mon[63965]: Deploying daemon osd.2 on smithi006 2024-09-17T10:54:08.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:08 smithi006 ceph-mon[63965]: pgmap v119: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 647 op/s; 0 B/s, 8 objects/s recovering 2024-09-17T10:54:08.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:08 smithi149 ceph-mon[49697]: Upgrade: Updating osd.2 2024-09-17T10:54:08.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:08 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:08.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:08 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T10:54:08.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:08 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:54:08.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:08 smithi149 ceph-mon[49697]: Deploying daemon osd.2 on smithi006 2024-09-17T10:54:08.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:08 smithi149 ceph-mon[49697]: pgmap v119: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 647 op/s; 0 B/s, 8 objects/s recovering 2024-09-17T10:54:08.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 197781 10.32 MB/sec execute 360 sec latency 40.504 ms 2024-09-17T10:54:09.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 198022 10.31 MB/sec execute 361 sec latency 107.805 ms 2024-09-17T10:54:10.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:10 smithi006 ceph-mon[63965]: pgmap v120: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 415 op/s 2024-09-17T10:54:10.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 198353 10.32 MB/sec execute 362 sec latency 60.742 ms 2024-09-17T10:54:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:10 smithi149 ceph-mon[49697]: pgmap v120: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 415 op/s 2024-09-17T10:54:11.739 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:11 smithi006 systemd[1]: Stopping Ceph osd.2 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:54:11.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 198597 10.31 MB/sec execute 363 sec latency 62.799 ms 2024-09-17T10:54:12.000 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:11 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[41963]: 2024-09-17T10:54:11.857+0000 7f737273b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:54:12.000 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:11 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[41963]: 2024-09-17T10:54:11.857+0000 7f737273b700 -1 osd.2 94 *** Got signal Terminated *** 2024-09-17T10:54:12.000 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:11 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[41963]: 2024-09-17T10:54:11.857+0000 7f737273b700 -1 osd.2 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T10:54:12.768 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:12 smithi006 ceph-mon[63965]: pgmap v121: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 517 op/s 2024-09-17T10:54:12.768 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:12 smithi006 ceph-mon[63965]: osd.2 marked itself down and dead 2024-09-17T10:54:12.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 198859 10.29 MB/sec execute 364 sec latency 183.721 ms 2024-09-17T10:54:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:12 smithi149 ceph-mon[49697]: pgmap v121: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 517 op/s 2024-09-17T10:54:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:12 smithi149 ceph-mon[49697]: osd.2 marked itself down and dead 2024-09-17T10:54:13.530 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:13 smithi006 podman[73074]: 2024-09-17 10:54:13.291505922 +0000 UTC m=+1.608142257 container died 60ca0fdb4a5189275c0c79c673b2ecebe15abc2efb3f8947efdf82294a05a18a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, architecture=x86_64, io.openshift.expose-services=, GIT_CLEAN=True, RELEASE=HEAD, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream) 2024-09-17T10:54:13.793 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:13 smithi006 ceph-mon[63965]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:54:13.793 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:13 smithi006 ceph-mon[63965]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T10:54:13.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 199120 10.27 MB/sec execute 365 sec latency 997.172 ms 2024-09-17T10:54:13.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:13 smithi149 ceph-mon[49697]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:54:13.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:13 smithi149 ceph-mon[49697]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T10:54:14.048 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:13 smithi006 podman[73074]: 2024-09-17 10:54:13.944435761 +0000 UTC m=+2.261072091 container cleanup 60ca0fdb4a5189275c0c79c673b2ecebe15abc2efb3f8947efdf82294a05a18a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, ceph=True, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, architecture=x86_64, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public) 2024-09-17T10:54:14.048 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:13 smithi006 bash[73074]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2 2024-09-17T10:54:14.559 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:14 smithi006 podman[73091]: 2024-09-17 10:54:14.300438001 +0000 UTC m=+1.002438789 container remove 60ca0fdb4a5189275c0c79c673b2ecebe15abc2efb3f8947efdf82294a05a18a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vcs-type=git, ceph=True, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , name=centos-stream, release=754, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_CLEAN=True) 2024-09-17T10:54:14.822 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:14 smithi006 ceph-mon[63965]: pgmap v123: 97 pgs: 2 peering, 7 stale+active+clean, 88 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 421 op/s 2024-09-17T10:54:14.822 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:14 smithi006 ceph-mon[63965]: osdmap e96: 8 total, 7 up, 8 in 2024-09-17T10:54:14.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 199401 10.25 MB/sec execute 366 sec latency 83.436 ms 2024-09-17T10:54:14.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:14 smithi149 ceph-mon[49697]: pgmap v123: 97 pgs: 2 peering, 7 stale+active+clean, 88 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 421 op/s 2024-09-17T10:54:14.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:14 smithi149 ceph-mon[49697]: osdmap e96: 8 total, 7 up, 8 in 2024-09-17T10:54:15.073 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:14 smithi006 podman[73179]: 2024-09-17 10:54:14.735912462 +0000 UTC m=+0.018598958 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:54:15.074 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:14 smithi006 podman[73179]: 2024-09-17 10:54:14.91610326 +0000 UTC m=+0.198789741 container create 882ab886bb794a2d3c3fb935008b85d7e163cd05488c6fd4bfae1da26294c06c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:54:15.325 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:15 smithi006 podman[73179]: 2024-09-17 10:54:15.286821711 +0000 UTC m=+0.569508211 container init 882ab886bb794a2d3c3fb935008b85d7e163cd05488c6fd4bfae1da26294c06c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True) 2024-09-17T10:54:15.325 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:15 smithi006 podman[73179]: 2024-09-17 10:54:15.2979836 +0000 UTC m=+0.580670089 container start 882ab886bb794a2d3c3fb935008b85d7e163cd05488c6fd4bfae1da26294c06c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T10:54:15.577 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:15 smithi006 podman[73179]: 2024-09-17 10:54:15.571110862 +0000 UTC m=+0.853797349 container attach 882ab886bb794a2d3c3fb935008b85d7e163cd05488c6fd4bfae1da26294c06c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-7498493, CEPH_POINT_RELEASE=) 2024-09-17T10:54:15.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 199800 10.26 MB/sec execute 367 sec latency 38.333 ms 2024-09-17T10:54:15.828 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:15 smithi006 podman[73179]: 2024-09-17 10:54:15.670411663 +0000 UTC m=+0.953098154 container died 882ab886bb794a2d3c3fb935008b85d7e163cd05488c6fd4bfae1da26294c06c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T10:54:16.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:16.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:54:16.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:15 smithi006 ceph-mon[63965]: pgmap v125: 97 pgs: 2 active+undersized, 2 peering, 5 stale+active+clean, 6 active+undersized+degraded, 82 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.1 MiB/s wr, 452 op/s; 32/1689 objects degraded (1.895%) 2024-09-17T10:54:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:54:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:15 smithi149 ceph-mon[49697]: pgmap v125: 97 pgs: 2 active+undersized, 2 peering, 5 stale+active+clean, 6 active+undersized+degraded, 82 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.1 MiB/s wr, 452 op/s; 32/1689 objects degraded (1.895%) 2024-09-17T10:54:16.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 200171 10.27 MB/sec execute 368 sec latency 46.332 ms 2024-09-17T10:54:16.865 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:16 smithi006 podman[73201]: 2024-09-17 10:54:16.729898221 +0000 UTC m=+1.050053717 container remove 882ab886bb794a2d3c3fb935008b85d7e163cd05488c6fd4bfae1da26294c06c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True) 2024-09-17T10:54:16.865 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:16 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-17T10:54:16.865 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:16 smithi006 systemd[1]: Stopped Ceph osd.2 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:54:16.865 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:16 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 55.692s CPU time. 2024-09-17T10:54:17.117 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:16 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 32/1689 objects degraded (1.895%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:17.117 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 systemd[1]: Starting Ceph osd.2 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:54:17.370 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 podman[73294]: 2024-09-17 10:54:17.214678225 +0000 UTC m=+0.020664296 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:54:17.371 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 podman[73294]: 2024-09-17 10:54:17.347841911 +0000 UTC m=+0.153827973 container create b5e1faf53135c3938535a9b8ebef503e0be1c61697c6c736f6b2c6be9d003498 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:54:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:16 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 32/1689 objects degraded (1.895%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:17.770 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 podman[73294]: 2024-09-17 10:54:17.687814813 +0000 UTC m=+0.493800885 container init b5e1faf53135c3938535a9b8ebef503e0be1c61697c6c736f6b2c6be9d003498 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T10:54:17.771 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 podman[73294]: 2024-09-17 10:54:17.699972622 +0000 UTC m=+0.505958682 container start b5e1faf53135c3938535a9b8ebef503e0be1c61697c6c736f6b2c6be9d003498 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-17T10:54:17.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 200610 10.26 MB/sec execute 369 sec latency 67.767 ms 2024-09-17T10:54:17.880 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 podman[73294]: 2024-09-17 10:54:17.869086787 +0000 UTC m=+0.675072846 container attach b5e1faf53135c3938535a9b8ebef503e0be1c61697c6c736f6b2c6be9d003498 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T10:54:18.048 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:18 smithi006 ceph-mon[63965]: pgmap v126: 97 pgs: 7 active+undersized, 2 peering, 18 active+undersized+degraded, 70 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 489 op/s; 149/1695 objects degraded (8.791%) 2024-09-17T10:54:18.048 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:54:18.048 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 bash[73294]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:54:18.048 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:54:18.049 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:17 smithi006 bash[73294]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:54:18.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:18 smithi149 ceph-mon[49697]: pgmap v126: 97 pgs: 7 active+undersized, 2 peering, 18 active+undersized+degraded, 70 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 489 op/s; 149/1695 objects degraded (8.791%) 2024-09-17T10:54:18.804 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:18 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T10:54:18.804 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:18 smithi006 bash[73294]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T10:54:18.804 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:18 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T10:54:18.804 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:18 smithi006 bash[73294]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T10:54:18.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 200854 10.25 MB/sec execute 370 sec latency 77.013 ms 2024-09-17T10:54:19.322 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T10:54:19.322 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 bash[73294]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T10:54:19.322 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T10:54:19.322 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 bash[73294]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T10:54:19.322 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T10:54:19.322 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 bash[73294]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T10:54:19.322 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T10:54:19.323 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 bash[73294]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T10:54:19.323 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate[73308]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T10:54:19.323 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 bash[73294]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T10:54:19.323 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:19 smithi006 podman[73294]: 2024-09-17 10:54:19.197055468 +0000 UTC m=+2.003041536 container died b5e1faf53135c3938535a9b8ebef503e0be1c61697c6c736f6b2c6be9d003498 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:54:19.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 201074 10.25 MB/sec execute 371 sec latency 67.620 ms 2024-09-17T10:54:20.604 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:20 smithi006 ceph-mon[63965]: pgmap v127: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.1 MiB/s wr, 308 op/s; 164/1707 objects degraded (9.607%) 2024-09-17T10:54:20.605 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:20 smithi006 podman[73443]: 2024-09-17 10:54:20.402111589 +0000 UTC m=+1.197833699 container remove b5e1faf53135c3938535a9b8ebef503e0be1c61697c6c736f6b2c6be9d003498 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T10:54:20.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 201345 10.25 MB/sec execute 372 sec latency 73.086 ms 2024-09-17T10:54:20.871 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:20 smithi006 podman[73493]: 2024-09-17 10:54:20.556290676 +0000 UTC m=+0.025093477 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:54:20.872 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:20 smithi006 podman[73493]: 2024-09-17 10:54:20.795909145 +0000 UTC m=+0.264711941 container create 8f61d308918e61591979e60eb830c97c2ddfddcb69fe4154e4bb4c6ec1c5c843 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T10:54:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:20 smithi149 ceph-mon[49697]: pgmap v127: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.1 MiB/s wr, 308 op/s; 164/1707 objects degraded (9.607%) 2024-09-17T10:54:21.391 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:21 smithi006 podman[73493]: 2024-09-17 10:54:21.315906857 +0000 UTC m=+0.784709658 container init 8f61d308918e61591979e60eb830c97c2ddfddcb69fe4154e4bb4c6ec1c5c843 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-17T10:54:21.391 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:21 smithi006 podman[73493]: 2024-09-17 10:54:21.329833517 +0000 UTC m=+0.798636307 container start 8f61d308918e61591979e60eb830c97c2ddfddcb69fe4154e4bb4c6ec1c5c843 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-7498493, org.label-schema.build-date=20240731) 2024-09-17T10:54:21.391 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:21 smithi006 ceph-osd[73510]: -- 172.21.15.6:0/2245590555 <== mon.0 v2:172.21.15.6:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5566c27bc1e0 con 0x5566c1957c00 2024-09-17T10:54:21.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:21 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 154/1635 objects degraded (9.419%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:21.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:21 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:21.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:21 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:21.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:21 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:54:21.648 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:21 smithi006 bash[73493]: 8f61d308918e61591979e60eb830c97c2ddfddcb69fe4154e4bb4c6ec1c5c843 2024-09-17T10:54:21.648 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:21 smithi006 systemd[1]: Started Ceph osd.2 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:54:21.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 201600 10.24 MB/sec execute 373 sec latency 117.913 ms 2024-09-17T10:54:21.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:21 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 154/1635 objects degraded (9.419%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:21.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:21 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:21.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:21 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:21.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:21 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:54:22.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:22 smithi006 ceph-mon[63965]: pgmap v128: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 493 op/s; 154/1635 objects degraded (9.419%) 2024-09-17T10:54:22.825 INFO:teuthology.orchestra.run.smithi006.stdout: 5 201942 10.24 MB/sec execute 374 sec latency 51.846 ms 2024-09-17T10:54:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:22 smithi149 ceph-mon[49697]: pgmap v128: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 493 op/s; 154/1635 objects degraded (9.419%) 2024-09-17T10:54:23.672 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:23 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[73506]: 2024-09-17T10:54:23.424+0000 7f0e0627f640 -1 Falling back to public interface 2024-09-17T10:54:23.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 202269 10.24 MB/sec execute 375 sec latency 80.890 ms 2024-09-17T10:54:24.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:24 smithi006 ceph-mon[63965]: pgmap v129: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 447 op/s; 150/1620 objects degraded (9.259%) 2024-09-17T10:54:24.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 202575 10.21 MB/sec execute 376 sec latency 168.800 ms 2024-09-17T10:54:24.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:24 smithi149 ceph-mon[49697]: pgmap v129: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 447 op/s; 150/1620 objects degraded (9.259%) 2024-09-17T10:54:25.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 202833 10.20 MB/sec execute 377 sec latency 81.011 ms 2024-09-17T10:54:26.241 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:26.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:26 smithi006 ceph-mon[63965]: pgmap v130: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.0 MiB/s wr, 452 op/s; 149/1641 objects degraded (9.080%) 2024-09-17T10:54:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:26.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:26 smithi149 ceph-mon[49697]: pgmap v130: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.0 MiB/s wr, 452 op/s; 149/1641 objects degraded (9.080%) 2024-09-17T10:54:26.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 203152 10.20 MB/sec execute 378 sec latency 58.394 ms 2024-09-17T10:54:27.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:27 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 149/1641 objects degraded (9.080%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:27.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:27 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 149/1641 objects degraded (9.080%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:27.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 203480 10.21 MB/sec execute 379 sec latency 47.276 ms 2024-09-17T10:54:28.309 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:28 smithi006 ceph-mon[63965]: pgmap v131: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 408 op/s; 149/1692 objects degraded (8.806%) 2024-09-17T10:54:28.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:28 smithi149 ceph-mon[49697]: pgmap v131: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 408 op/s; 149/1692 objects degraded (8.806%) 2024-09-17T10:54:28.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 203909 10.20 MB/sec execute 380 sec latency 68.271 ms 2024-09-17T10:54:29.790 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:54:29.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 204265 10.20 MB/sec execute 381 sec latency 53.682 ms 2024-09-17T10:54:30.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:30 smithi006 ceph-mon[63965]: pgmap v132: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.1 MiB/s wr, 382 op/s; 149/1656 objects degraded (8.998%) 2024-09-17T10:54:30.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:30 smithi006 ceph-mon[63965]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:54:30.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:54:30.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:30 smithi149 ceph-mon[49697]: pgmap v132: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.1 MiB/s wr, 382 op/s; 149/1656 objects degraded (8.998%) 2024-09-17T10:54:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:30 smithi149 ceph-mon[49697]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:54:30.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:54:30.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:30.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 204519 10.19 MB/sec execute 382 sec latency 57.538 ms 2024-09-17T10:54:30.876 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (5m) 1s ago 14m 17.7M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (2m) 1s ago 14m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (2m) 2m ago 14m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (5m) 1s ago 14m 43.5M - dad864ee21e9 89df198e1a9b 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (8m) 1s ago 8m 38.0M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (8m) 2m ago 8m 14.7M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (4m) 1s ago 16m 517M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (3m) 2m ago 14m 453M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (3m) 1s ago 16m 50.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:54:30.877 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (3m) 2m ago 14m 35.2M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (8m) 1s ago 8m 287M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (8m) 2m ago 8m 86.0M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (5m) 1s ago 14m 9521k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (5m) 2m ago 14m 11.5M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (105s) 1s ago 13m 294M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (57s) 1s ago 12m 217M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (10s) 1s ago 12m 12.2M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (12m) 1s ago 12m 371M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (11m) 2m ago 11m 513M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (11m) 2m ago 11m 334M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (10m) 2m ago 10m 530M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:54:30.878 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (10m) 2m ago 10m 391M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:54:30.879 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (4m) 1s ago 13m 49.0M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:54:31.345 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:31 smithi006 ceph-mon[63965]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:54:31.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:31 smithi006 ceph-mon[63965]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:54:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:31 smithi149 ceph-mon[49697]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:54:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:31 smithi149 ceph-mon[49697]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:54:31.532 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:54:31.532 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:54:31.532 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:54:31.532 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:54:31.532 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:54:31.532 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 6 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:54:31.533 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:54:31.597 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:31 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[73506]: 2024-09-17T10:54:31.479+0000 7f0e0627f640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-17T10:54:31.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 204946 10.21 MB/sec execute 383 sec latency 54.261 ms 2024-09-17T10:54:32.092 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:54:32.092 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:54:32.092 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:54:32.092 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:54:32.092 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:54:32.093 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:54:32.093 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:54:32.093 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:54:32.093 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:54:32.093 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "9/23 daemons upgraded", 2024-09-17T10:54:32.093 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:54:32.093 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:54:32.093 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:54:32.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:32 smithi006 ceph-mon[63965]: pgmap v133: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 550 op/s; 144/1647 objects degraded (8.743%) 2024-09-17T10:54:32.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:32 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2473629820' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:54:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:32 smithi149 ceph-mon[49697]: pgmap v133: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 550 op/s; 144/1647 objects degraded (8.743%) 2024-09-17T10:54:32.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:32 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2473629820' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:54:32.624 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:32 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[73506]: 2024-09-17T10:54:32.543+0000 7f0e0627f640 -1 osd.2 94 log_to_monitors true 2024-09-17T10:54:32.732 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 144/1647 objects degraded (8.743%), 19 pgs degraded 2024-09-17T10:54:32.732 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: osd.2 (root=default,host=smithi006) is down 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 144/1647 objects degraded (8.743%), 19 pgs degraded 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-17T10:54:32.733 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-17T10:54:32.734 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-17T10:54:32.734 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-17T10:54:32.734 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-17T10:54:32.734 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-17T10:54:32.734 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-17T10:54:32.734 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-17T10:54:32.734 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-17T10:54:32.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 205265 10.20 MB/sec execute 384 sec latency 91.443 ms 2024-09-17T10:54:33.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:33 smithi006 ceph-mon[63965]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:54:33.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:33 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 144/1647 objects degraded (8.743%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:33.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:33 smithi006 ceph-mon[63965]: from='osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-17T10:54:33.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:33 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3193331991' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:54:33.735 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:33 smithi149 ceph-mon[49697]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:54:33.735 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:33 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 144/1647 objects degraded (8.743%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:33.735 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:33 smithi149 ceph-mon[49697]: from='osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-17T10:54:33.735 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:33 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3193331991' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:54:33.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 205560 10.21 MB/sec execute 385 sec latency 92.305 ms 2024-09-17T10:54:33.926 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 10:54:33 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[73506]: 2024-09-17T10:54:33.720+0000 7f0dfa021640 -1 osd.2 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:54:34.178 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:34 smithi006 ceph-mon[63965]: from='osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T10:54:34.179 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:34 smithi006 ceph-mon[63965]: osdmap e97: 8 total, 7 up, 8 in 2024-09-17T10:54:34.179 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:34 smithi006 ceph-mon[63965]: from='osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]} : dispatch 2024-09-17T10:54:34.179 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:34 smithi006 ceph-mon[63965]: pgmap v135: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 463 op/s; 149/1704 objects degraded (8.744%) 2024-09-17T10:54:34.179 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:34 smithi006 ceph-mon[63965]: from='osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070]' entity='osd.2' 2024-09-17T10:54:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:34 smithi149 ceph-mon[49697]: from='osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T10:54:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:34 smithi149 ceph-mon[49697]: osdmap e97: 8 total, 7 up, 8 in 2024-09-17T10:54:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:34 smithi149 ceph-mon[49697]: from='osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]} : dispatch 2024-09-17T10:54:34.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:34 smithi149 ceph-mon[49697]: pgmap v135: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 463 op/s; 149/1704 objects degraded (8.744%) 2024-09-17T10:54:34.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:34 smithi149 ceph-mon[49697]: from='osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070]' entity='osd.2' 2024-09-17T10:54:34.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 205742 10.19 MB/sec execute 386 sec latency 92.599 ms 2024-09-17T10:54:35.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:35 smithi006 ceph-mon[63965]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:54:35.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:35 smithi006 ceph-mon[63965]: osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070] boot 2024-09-17T10:54:35.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:35 smithi006 ceph-mon[63965]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T10:54:35.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:35 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T10:54:35.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:35 smithi149 ceph-mon[49697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:54:35.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:35 smithi149 ceph-mon[49697]: osd.2 [v2:172.21.15.6:6818/1324652070,v1:172.21.15.6:6819/1324652070] boot 2024-09-17T10:54:35.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:35 smithi149 ceph-mon[49697]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T10:54:35.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:35 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T10:54:35.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 205900 10.17 MB/sec execute 387 sec latency 650.072 ms 2024-09-17T10:54:36.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:36 smithi006 ceph-mon[63965]: pgmap v137: 97 pgs: 6 peering, 7 active+undersized, 14 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.5 MiB/s wr, 516 op/s; 122/1665 objects degraded (7.327%) 2024-09-17T10:54:36.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:36 smithi006 ceph-mon[63965]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T10:54:36.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:36.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:36.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:54:36.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:54:36.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:36.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:36.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 206121 10.15 MB/sec execute 388 sec latency 1587.931 ms 2024-09-17T10:54:36.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:36 smithi149 ceph-mon[49697]: pgmap v137: 97 pgs: 6 peering, 7 active+undersized, 14 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.5 MiB/s wr, 516 op/s; 122/1665 objects degraded (7.327%) 2024-09-17T10:54:36.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:36 smithi149 ceph-mon[49697]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T10:54:36.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:36.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:36.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:54:36.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:54:36.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:36.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-17T10:54:37.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 122/1665 objects degraded (7.327%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:37.826 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:37 smithi006 ceph-mon[63965]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T10:54:37.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 206375 10.13 MB/sec execute 389 sec latency 533.889 ms 2024-09-17T10:54:37.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T10:54:37.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:54:37.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:54:37.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:54:37.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:54:37.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T10:54:37.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T10:54:37.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-17T10:54:37.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 122/1665 objects degraded (7.327%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:37.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:37 smithi149 ceph-mon[49697]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T10:54:38.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 206576 10.12 MB/sec execute 390 sec latency 969.671 ms 2024-09-17T10:54:38.852 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:38 smithi006 ceph-mon[63965]: pgmap v141: 97 pgs: 1 active+undersized+remapped, 14 peering, 3 active+undersized, 6 active+undersized+degraded, 73 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 5.1 MiB/s wr, 494 op/s; 40/1728 objects degraded (2.315%); 134 KiB/s, 8 objects/s recovering 2024-09-17T10:54:38.852 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:38 smithi006 ceph-mon[63965]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T10:54:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:38 smithi149 ceph-mon[49697]: pgmap v141: 97 pgs: 1 active+undersized+remapped, 14 peering, 3 active+undersized, 6 active+undersized+degraded, 73 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 5.1 MiB/s wr, 494 op/s; 40/1728 objects degraded (2.315%); 134 KiB/s, 8 objects/s recovering 2024-09-17T10:54:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:38 smithi149 ceph-mon[49697]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T10:54:39.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 206899 10.13 MB/sec execute 391 sec latency 80.835 ms 2024-09-17T10:54:40.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:40 smithi006 ceph-mon[63965]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T10:54:40.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:40 smithi006 ceph-mon[63965]: pgmap v144: 97 pgs: 1 active+undersized+remapped, 15 peering, 81 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 837 KiB/s wr, 52 op/s; 374 KiB/s, 15 objects/s recovering 2024-09-17T10:54:40.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:40 smithi149 ceph-mon[49697]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T10:54:40.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:40 smithi149 ceph-mon[49697]: pgmap v144: 97 pgs: 1 active+undersized+remapped, 15 peering, 81 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 837 KiB/s wr, 52 op/s; 374 KiB/s, 15 objects/s recovering 2024-09-17T10:54:40.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 207261 10.12 MB/sec execute 392 sec latency 208.871 ms 2024-09-17T10:54:41.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:41 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/1728 objects degraded (2.315%), 6 pgs degraded) 2024-09-17T10:54:41.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:41 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:54:41.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:41 smithi006 ceph-mon[63965]: osdmap e104: 8 total, 8 up, 8 in 2024-09-17T10:54:41.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:41 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/1728 objects degraded (2.315%), 6 pgs degraded) 2024-09-17T10:54:41.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:41 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:54:41.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:41 smithi149 ceph-mon[49697]: osdmap e104: 8 total, 8 up, 8 in 2024-09-17T10:54:41.826 INFO:teuthology.orchestra.run.smithi006.stdout: 5 207518 10.11 MB/sec execute 393 sec latency 683.713 ms 2024-09-17T10:54:42.532 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:42 smithi006 ceph-mon[63965]: osdmap e105: 8 total, 8 up, 8 in 2024-09-17T10:54:42.532 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:42 smithi006 ceph-mon[63965]: pgmap v147: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 peering, 91 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.9 MiB/s wr, 231 op/s; 29/1593 objects degraded (1.820%); 2.2 MiB/s, 33 objects/s recovering 2024-09-17T10:54:42.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:42 smithi149 ceph-mon[49697]: osdmap e105: 8 total, 8 up, 8 in 2024-09-17T10:54:42.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:42 smithi149 ceph-mon[49697]: pgmap v147: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 peering, 91 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.9 MiB/s wr, 231 op/s; 29/1593 objects degraded (1.820%); 2.2 MiB/s, 33 objects/s recovering 2024-09-17T10:54:42.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 207769 10.10 MB/sec execute 394 sec latency 748.827 ms 2024-09-17T10:54:43.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:43 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 29/1593 objects degraded (1.820%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:43.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:43 smithi006 ceph-mon[63965]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T10:54:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:43 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 29/1593 objects degraded (1.820%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T10:54:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:43 smithi149 ceph-mon[49697]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T10:54:43.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 208040 10.10 MB/sec execute 395 sec latency 41.052 ms 2024-09-17T10:54:44.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:44 smithi006 ceph-mon[63965]: pgmap v149: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 93 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 543 op/s; 29/1569 objects degraded (1.848%); 1.9 MiB/s, 25 objects/s recovering 2024-09-17T10:54:44.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:44 smithi149 ceph-mon[49697]: pgmap v149: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 93 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 543 op/s; 29/1569 objects degraded (1.848%); 1.9 MiB/s, 25 objects/s recovering 2024-09-17T10:54:44.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 208416 10.11 MB/sec execute 396 sec latency 42.968 ms 2024-09-17T10:54:45.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 208731 10.11 MB/sec execute 397 sec latency 56.226 ms 2024-09-17T10:54:46.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:46.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:54:46.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:46 smithi006 ceph-mon[63965]: pgmap v150: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.8 MiB/s wr, 423 op/s; 1.3 MiB/s, 23 objects/s recovering 2024-09-17T10:54:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:54:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:46 smithi149 ceph-mon[49697]: pgmap v150: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.8 MiB/s wr, 423 op/s; 1.3 MiB/s, 23 objects/s recovering 2024-09-17T10:54:46.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 208966 10.11 MB/sec execute 398 sec latency 57.392 ms 2024-09-17T10:54:47.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:47 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1569 objects degraded (1.848%), 3 pgs degraded) 2024-09-17T10:54:47.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:47 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:54:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:47 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1569 objects degraded (1.848%), 3 pgs degraded) 2024-09-17T10:54:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:47 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:54:47.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 209183 10.10 MB/sec execute 399 sec latency 51.991 ms 2024-09-17T10:54:48.194 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:48 smithi006 ceph-mon[63965]: pgmap v151: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.0 MiB/s wr, 534 op/s; 1.1 MiB/s, 19 objects/s recovering 2024-09-17T10:54:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:48 smithi149 ceph-mon[49697]: pgmap v151: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.0 MiB/s wr, 534 op/s; 1.1 MiB/s, 19 objects/s recovering 2024-09-17T10:54:48.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 209419 10.07 MB/sec execute 400 sec latency 60.471 ms 2024-09-17T10:54:49.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 209704 10.06 MB/sec execute 401 sec latency 44.791 ms 2024-09-17T10:54:50.730 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:50 smithi006 ceph-mon[63965]: pgmap v152: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 471 op/s; 0 B/s, 3 objects/s recovering 2024-09-17T10:54:50.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 209931 10.05 MB/sec execute 402 sec latency 57.633 ms 2024-09-17T10:54:50.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:50 smithi149 ceph-mon[49697]: pgmap v152: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 471 op/s; 0 B/s, 3 objects/s recovering 2024-09-17T10:54:51.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 210158 10.03 MB/sec execute 403 sec latency 68.381 ms 2024-09-17T10:54:52.779 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:52 smithi006 ceph-mon[63965]: pgmap v153: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 461 op/s; 0 B/s, 2 objects/s recovering 2024-09-17T10:54:52.779 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T10:54:52.779 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:52 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T10:54:52.779 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:52 smithi006 ceph-mon[63965]: Upgrade: osd.3 is safe to restart 2024-09-17T10:54:52.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 210457 10.03 MB/sec execute 404 sec latency 45.163 ms 2024-09-17T10:54:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:52 smithi149 ceph-mon[49697]: pgmap v153: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 461 op/s; 0 B/s, 2 objects/s recovering 2024-09-17T10:54:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T10:54:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:52 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T10:54:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:52 smithi149 ceph-mon[49697]: Upgrade: osd.3 is safe to restart 2024-09-17T10:54:53.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 210817 10.03 MB/sec execute 405 sec latency 56.052 ms 2024-09-17T10:54:54.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 211264 10.03 MB/sec execute 406 sec latency 41.025 ms 2024-09-17T10:54:54.832 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:54 smithi006 ceph-mon[63965]: pgmap v154: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 3.9 MiB/s wr, 347 op/s; 0 B/s, 2 objects/s recovering 2024-09-17T10:54:54.832 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:54.833 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T10:54:54.833 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:54 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:54:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:54 smithi149 ceph-mon[49697]: pgmap v154: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 3.9 MiB/s wr, 347 op/s; 0 B/s, 2 objects/s recovering 2024-09-17T10:54:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T10:54:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:54 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:54:55.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:55 smithi006 ceph-mon[63965]: Upgrade: Updating osd.3 2024-09-17T10:54:55.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:55 smithi006 ceph-mon[63965]: Deploying daemon osd.3 on smithi006 2024-09-17T10:54:55.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:55.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 211506 10.03 MB/sec execute 407 sec latency 49.837 ms 2024-09-17T10:54:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:55 smithi149 ceph-mon[49697]: Upgrade: Updating osd.3 2024-09-17T10:54:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:55 smithi149 ceph-mon[49697]: Deploying daemon osd.3 on smithi006 2024-09-17T10:54:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:54:56.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:56 smithi006 ceph-mon[63965]: pgmap v155: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 440 op/s; 0 B/s, 2 objects/s recovering 2024-09-17T10:54:56.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 211763 10.03 MB/sec execute 408 sec latency 43.706 ms 2024-09-17T10:54:56.857 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:56 smithi149 ceph-mon[49697]: pgmap v155: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 440 op/s; 0 B/s, 2 objects/s recovering 2024-09-17T10:54:57.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 212092 10.03 MB/sec execute 409 sec latency 53.729 ms 2024-09-17T10:54:58.828 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:54:58 smithi006 ceph-mon[63965]: pgmap v156: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 456 op/s 2024-09-17T10:54:58.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 212405 10.03 MB/sec execute 410 sec latency 52.127 ms 2024-09-17T10:54:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:54:58 smithi149 ceph-mon[49697]: pgmap v156: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 456 op/s 2024-09-17T10:54:59.330 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:54:59 smithi006 systemd[1]: Stopping Ceph osd.3 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:54:59.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 212680 10.03 MB/sec execute 411 sec latency 49.997 ms 2024-09-17T10:54:59.833 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:54:59 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[45191]: 2024-09-17T10:54:59.662+0000 7fb0a08a6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:54:59.833 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:54:59 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[45191]: 2024-09-17T10:54:59.662+0000 7fb0a08a6700 -1 osd.3 106 *** Got signal Terminated *** 2024-09-17T10:54:59.833 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:54:59 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[45191]: 2024-09-17T10:54:59.662+0000 7fb0a08a6700 -1 osd.3 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T10:55:00.827 INFO:teuthology.orchestra.run.smithi006.stdout: 5 212921 10.01 MB/sec execute 412 sec latency 174.452 ms 2024-09-17T10:55:00.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:00 smithi006 ceph-mon[63965]: pgmap v157: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.6 MiB/s wr, 360 op/s 2024-09-17T10:55:00.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:00 smithi006 ceph-mon[63965]: osd.3 marked itself down and dead 2024-09-17T10:55:00.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:00.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:55:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:00 smithi149 ceph-mon[49697]: pgmap v157: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.6 MiB/s wr, 360 op/s 2024-09-17T10:55:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:00 smithi149 ceph-mon[49697]: osd.3 marked itself down and dead 2024-09-17T10:55:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:00.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:55:01.645 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:01 smithi006 podman[75419]: 2024-09-17 10:55:01.307902989 +0000 UTC m=+1.845711044 container died 856f69a106bac7f1745539f41ba3a78b64914c563ac9597100be01a5ac8a59e3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, name=centos-stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, ceph=True) 2024-09-17T10:55:01.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 212962 9.99 MB/sec execute 413 sec latency 1020.661 ms 2024-09-17T10:55:01.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:01 smithi006 ceph-mon[63965]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:55:01.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:01 smithi006 ceph-mon[63965]: osdmap e107: 8 total, 7 up, 8 in 2024-09-17T10:55:01.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:01 smithi006 ceph-mon[63965]: osdmap e108: 8 total, 7 up, 8 in 2024-09-17T10:55:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:01 smithi149 ceph-mon[49697]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:55:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:01 smithi149 ceph-mon[49697]: osdmap e107: 8 total, 7 up, 8 in 2024-09-17T10:55:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:01 smithi149 ceph-mon[49697]: osdmap e108: 8 total, 7 up, 8 in 2024-09-17T10:55:02.402 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:02 smithi006 podman[75419]: 2024-09-17 10:55:02.150584488 +0000 UTC m=+2.688392548 container cleanup 856f69a106bac7f1745539f41ba3a78b64914c563ac9597100be01a5ac8a59e3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, name=centos-stream, GIT_BRANCH=HEAD, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, ceph=True, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, vcs-type=git, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8) 2024-09-17T10:55:02.402 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:02 smithi006 bash[75419]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3 2024-09-17T10:55:02.402 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:02 smithi006 podman[75436]: 2024-09-17 10:55:02.394177115 +0000 UTC m=+1.089062844 container remove 856f69a106bac7f1745539f41ba3a78b64914c563ac9597100be01a5ac8a59e3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, release=754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, ceph=True, version=8, vendor=Red Hat, Inc.) 2024-09-17T10:55:02.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:02 smithi006 ceph-mon[63965]: pgmap v159: 97 pgs: 16 stale+active+clean, 81 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 387 op/s 2024-09-17T10:55:02.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 213201 9.97 MB/sec execute 414 sec latency 62.526 ms 2024-09-17T10:55:02.921 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:02 smithi006 podman[75519]: 2024-09-17 10:55:02.811741624 +0000 UTC m=+0.025454614 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:55:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:02 smithi149 ceph-mon[49697]: pgmap v159: 97 pgs: 16 stale+active+clean, 81 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 387 op/s 2024-09-17T10:55:03.171 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:02 smithi006 podman[75519]: 2024-09-17 10:55:02.920627986 +0000 UTC m=+0.134340971 container create 33f2d0f1ef5bcaa29beae3dcede49c8c0702d24490a07651c131115dcfda0269 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-17T10:55:03.251 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:55:03.425 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:03 smithi006 podman[75519]: 2024-09-17 10:55:03.315933071 +0000 UTC m=+0.529646050 container init 33f2d0f1ef5bcaa29beae3dcede49c8c0702d24490a07651c131115dcfda0269 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T10:55:03.425 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:03 smithi006 podman[75519]: 2024-09-17 10:55:03.323478019 +0000 UTC m=+0.537190997 container start 33f2d0f1ef5bcaa29beae3dcede49c8c0702d24490a07651c131115dcfda0269 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:55:03.690 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:03 smithi006 podman[75519]: 2024-09-17 10:55:03.424352034 +0000 UTC m=+0.638065017 container attach 33f2d0f1ef5bcaa29beae3dcede49c8c0702d24490a07651c131115dcfda0269 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:55:03.690 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:03 smithi006 podman[75519]: 2024-09-17 10:55:03.672914418 +0000 UTC m=+0.886627408 container died 33f2d0f1ef5bcaa29beae3dcede49c8c0702d24490a07651c131115dcfda0269 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-17T10:55:03.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 213458 9.96 MB/sec execute 415 sec latency 53.455 ms 2024-09-17T10:55:03.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:03 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 107/1632 objects degraded (6.556%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:03.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:03 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 107/1632 objects degraded (6.556%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:04.142 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:55:04.142 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (6m) 34s ago 15m 17.7M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:55:04.142 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (2m) 34s ago 15m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:55:04.142 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (2m) 2m ago 14m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:55:04.142 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (6m) 34s ago 15m 43.5M - dad864ee21e9 89df198e1a9b 2024-09-17T10:55:04.142 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (9m) 34s ago 9m 38.0M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (9m) 2m ago 9m 14.7M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (5m) 34s ago 16m 517M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (4m) 34s ago 16m 50.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (3m) 2m ago 14m 35.2M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (9m) 34s ago 9m 287M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (9m) 2m ago 9m 86.0M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (6m) 34s ago 15m 9521k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (5m) 2m ago 14m 11.5M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (2m) 34s ago 13m 294M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (90s) 34s ago 13m 217M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (43s) 34s ago 12m 12.2M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:55:04.143 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (12m) 34s ago 12m 371M 1937M 17.2.0 e1d6a67b021e 856f69a106ba 2024-09-17T10:55:04.144 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (12m) 2m ago 12m 513M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:55:04.144 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (11m) 2m ago 11m 334M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:55:04.144 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (11m) 2m ago 11m 530M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:55:04.144 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (11m) 2m ago 11m 391M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:55:04.144 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (4m) 34s ago 14m 49.0M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:55:04.690 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:55:04.690 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:55:04.690 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:55:04.690 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:55:04.690 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:55:04.690 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 3 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 7 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:55:04.691 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:55:04.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:04 smithi006 ceph-mon[63965]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:04.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:04 smithi006 ceph-mon[63965]: pgmap v161: 97 pgs: 7 active+undersized, 8 stale+active+clean, 16 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 344 op/s; 107/1632 objects degraded (6.556%) 2024-09-17T10:55:04.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:04 smithi006 ceph-mon[63965]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:04.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:04 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3386906304' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:04.702 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:04 smithi006 podman[75587]: 2024-09-17 10:55:04.556863837 +0000 UTC m=+0.873572926 container remove 33f2d0f1ef5bcaa29beae3dcede49c8c0702d24490a07651c131115dcfda0269 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-7498493, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-17T10:55:04.702 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:04 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-17T10:55:04.702 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:04 smithi006 systemd[1]: Stopped Ceph osd.3 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:55:04.702 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:04 smithi006 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.3.service: Consumed 4min 16.407s CPU time. 2024-09-17T10:55:04.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 213680 9.95 MB/sec execute 416 sec latency 81.516 ms 2024-09-17T10:55:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:04 smithi149 ceph-mon[49697]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:04 smithi149 ceph-mon[49697]: pgmap v161: 97 pgs: 7 active+undersized, 8 stale+active+clean, 16 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 344 op/s; 107/1632 objects degraded (6.556%) 2024-09-17T10:55:04.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:04 smithi149 ceph-mon[49697]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:04.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:04 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3386906304' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:04.958 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:04 smithi006 systemd[1]: Starting Ceph osd.3 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:55:05.209 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:55:05.209 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:55:05.209 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:55:05.209 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:55:05.209 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:55:05.209 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:55:05.209 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:55:05.210 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:55:05.210 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:55:05.210 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "9/23 daemons upgraded", 2024-09-17T10:55:05.210 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:55:05.210 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:55:05.210 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:55:05.217 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 podman[75748]: 2024-09-17 10:55:05.079115194 +0000 UTC m=+0.026722506 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:55:05.217 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 podman[75748]: 2024-09-17 10:55:05.208573952 +0000 UTC m=+0.156181256 container create 33a16d1ec056467da16bf84ac5f7e1afa2965340f1b00048ba70cdb7de2d715e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-17T10:55:05.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:05 smithi006 ceph-mon[63965]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:05.731 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 podman[75748]: 2024-09-17 10:55:05.562965808 +0000 UTC m=+0.510573119 container init 33a16d1ec056467da16bf84ac5f7e1afa2965340f1b00048ba70cdb7de2d715e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-7498493, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:55:05.731 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 podman[75748]: 2024-09-17 10:55:05.578761746 +0000 UTC m=+0.526369051 container start 33a16d1ec056467da16bf84ac5f7e1afa2965340f1b00048ba70cdb7de2d715e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=main-7498493, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-17T10:55:05.731 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 podman[75748]: 2024-09-17 10:55:05.730001284 +0000 UTC m=+0.677608593 container attach 33a16d1ec056467da16bf84ac5f7e1afa2965340f1b00048ba70cdb7de2d715e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 179/1605 objects degraded (11.153%), 23 pgs degraded 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout: osd.3 (root=default,host=smithi006) is down 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 179/1605 objects degraded (11.153%), 23 pgs degraded 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-09-17T10:55:05.774 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-17T10:55:05.775 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-09-17T10:55:05.776 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-09-17T10:55:05.776 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-09-17T10:55:05.776 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-17T10:55:05.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 213963 9.94 MB/sec execute 417 sec latency 62.071 ms 2024-09-17T10:55:05.982 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:55:05.982 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 bash[75748]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:55:05.982 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:55:05.982 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:05 smithi006 bash[75748]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:55:06.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:05 smithi149 ceph-mon[49697]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:06.533 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T10:55:06.533 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 bash[75748]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T10:55:06.533 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T10:55:06.533 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 bash[75748]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T10:55:06.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-mon[63965]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:06.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-mon[63965]: pgmap v162: 97 pgs: 12 active+undersized, 5 stale+active+clean, 23 active+undersized+degraded, 57 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.4 MiB/s wr, 442 op/s; 179/1605 objects degraded (11.153%) 2024-09-17T10:55:06.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2428358506' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:55:06.791 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T10:55:06.791 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 bash[75748]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T10:55:06.791 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T10:55:06.791 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 bash[75748]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T10:55:06.791 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T10:55:06.791 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 bash[75748]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T10:55:06.792 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T10:55:06.792 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 bash[75748]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T10:55:06.792 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate[75784]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T10:55:06.792 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 bash[75748]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T10:55:06.792 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:06 smithi006 podman[75748]: 2024-09-17 10:55:06.778422236 +0000 UTC m=+1.726029549 container died 33a16d1ec056467da16bf84ac5f7e1afa2965340f1b00048ba70cdb7de2d715e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-7498493) 2024-09-17T10:55:06.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 214307 9.94 MB/sec execute 418 sec latency 94.664 ms 2024-09-17T10:55:07.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:06 smithi149 ceph-mon[49697]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:07.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:06 smithi149 ceph-mon[49697]: pgmap v162: 97 pgs: 12 active+undersized, 5 stale+active+clean, 23 active+undersized+degraded, 57 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.4 MiB/s wr, 442 op/s; 179/1605 objects degraded (11.153%) 2024-09-17T10:55:07.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:06 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2428358506' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:55:07.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 214606 9.93 MB/sec execute 419 sec latency 77.675 ms 2024-09-17T10:55:08.065 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:07 smithi006 podman[75929]: 2024-09-17 10:55:07.859303485 +0000 UTC m=+1.071669913 container remove 33a16d1ec056467da16bf84ac5f7e1afa2965340f1b00048ba70cdb7de2d715e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T10:55:08.317 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:08 smithi006 podman[75988]: 2024-09-17 10:55:07.992589019 +0000 UTC m=+0.020281147 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:55:08.317 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:08 smithi006 podman[75988]: 2024-09-17 10:55:08.109692906 +0000 UTC m=+0.137385037 container create 654bf1272724d1afe4d81d46fbfb4ae9d2319ac96004b37cc0283d06a2cd3b35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-7498493) 2024-09-17T10:55:08.583 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:08 smithi006 podman[75988]: 2024-09-17 10:55:08.451041999 +0000 UTC m=+0.478734132 container init 654bf1272724d1afe4d81d46fbfb4ae9d2319ac96004b37cc0283d06a2cd3b35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-7498493) 2024-09-17T10:55:08.583 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:08 smithi006 podman[75988]: 2024-09-17 10:55:08.455854963 +0000 UTC m=+0.483547093 container start 654bf1272724d1afe4d81d46fbfb4ae9d2319ac96004b37cc0283d06a2cd3b35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-17T10:55:08.583 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:08 smithi006 ceph-osd[76006]: -- 172.21.15.6:0/1933708856 <== mon.0 v2:172.21.15.6:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55990f8be1e0 con 0x55990ea59c00 2024-09-17T10:55:08.583 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:08 smithi006 bash[75988]: 654bf1272724d1afe4d81d46fbfb4ae9d2319ac96004b37cc0283d06a2cd3b35 2024-09-17T10:55:08.583 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:08 smithi006 systemd[1]: Started Ceph osd.3 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:55:08.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 214890 9.93 MB/sec execute 420 sec latency 74.806 ms 2024-09-17T10:55:08.847 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:08 smithi006 ceph-mon[63965]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 449 op/s; 212/1590 objects degraded (13.333%) 2024-09-17T10:55:08.847 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:08.847 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:08.847 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:08 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:55:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:08 smithi149 ceph-mon[49697]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 449 op/s; 212/1590 objects degraded (13.333%) 2024-09-17T10:55:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:08 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:08 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:08 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:55:09.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 215171 9.93 MB/sec execute 421 sec latency 47.916 ms 2024-09-17T10:55:09.880 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:09 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 212/1599 objects degraded (13.258%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:10.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:09 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 212/1599 objects degraded (13.258%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:10.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 215412 9.93 MB/sec execute 422 sec latency 69.685 ms 2024-09-17T10:55:10.945 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:10 smithi006 ceph-mon[63965]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.4 MiB/s wr, 303 op/s; 212/1599 objects degraded (13.258%) 2024-09-17T10:55:10.945 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:10 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[76002]: 2024-09-17T10:55:10.750+0000 7ff55f1ac640 -1 Falling back to public interface 2024-09-17T10:55:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:10 smithi149 ceph-mon[49697]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.4 MiB/s wr, 303 op/s; 212/1599 objects degraded (13.258%) 2024-09-17T10:55:11.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 215668 9.92 MB/sec execute 423 sec latency 56.118 ms 2024-09-17T10:55:12.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 215989 9.93 MB/sec execute 424 sec latency 61.192 ms 2024-09-17T10:55:13.018 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:12 smithi006 ceph-mon[63965]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.6 MiB/s wr, 434 op/s; 223/1629 objects degraded (13.689%) 2024-09-17T10:55:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:12 smithi149 ceph-mon[49697]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.6 MiB/s wr, 434 op/s; 223/1629 objects degraded (13.689%) 2024-09-17T10:55:13.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 216205 9.92 MB/sec execute 425 sec latency 85.157 ms 2024-09-17T10:55:14.798 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:14 smithi006 ceph-mon[63965]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.5 MiB/s wr, 385 op/s; 233/1671 objects degraded (13.944%) 2024-09-17T10:55:14.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 216372 9.91 MB/sec execute 426 sec latency 76.829 ms 2024-09-17T10:55:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:14 smithi149 ceph-mon[49697]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.5 MiB/s wr, 385 op/s; 233/1671 objects degraded (13.944%) 2024-09-17T10:55:15.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 216605 9.89 MB/sec execute 427 sec latency 53.543 ms 2024-09-17T10:55:16.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:16.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:55:16.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:16 smithi006 ceph-mon[63965]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.8 MiB/s wr, 400 op/s; 230/1701 objects degraded (13.521%) 2024-09-17T10:55:16.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:16.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:16.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:55:16.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:16 smithi149 ceph-mon[49697]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.8 MiB/s wr, 400 op/s; 230/1701 objects degraded (13.521%) 2024-09-17T10:55:16.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:16.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:16.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 216813 9.87 MB/sec execute 428 sec latency 84.185 ms 2024-09-17T10:55:17.122 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:17 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 230/1701 objects degraded (13.521%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:17.123 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:17 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:17.123 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:17 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:17 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 230/1701 objects degraded (13.521%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:17 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:17 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:17.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 217048 9.86 MB/sec execute 429 sec latency 68.524 ms 2024-09-17T10:55:18.145 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:18 smithi006 ceph-mon[63965]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.7 MiB/s wr, 333 op/s; 225/1683 objects degraded (13.369%) 2024-09-17T10:55:18.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:18 smithi149 ceph-mon[49697]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.7 MiB/s wr, 333 op/s; 225/1683 objects degraded (13.369%) 2024-09-17T10:55:18.828 INFO:teuthology.orchestra.run.smithi006.stdout: 5 217272 9.84 MB/sec execute 430 sec latency 47.520 ms 2024-09-17T10:55:18.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:18 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[76002]: 2024-09-17T10:55:18.675+0000 7ff55f1ac640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-17T10:55:19.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 217575 9.85 MB/sec execute 431 sec latency 98.674 ms 2024-09-17T10:55:20.451 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:20 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[76002]: 2024-09-17T10:55:20.248+0000 7ff55f1ac640 -1 osd.3 106 log_to_monitors true 2024-09-17T10:55:20.725 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:20 smithi006 ceph-mon[63965]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.6 MiB/s wr, 327 op/s; 222/1671 objects degraded (13.285%) 2024-09-17T10:55:20.725 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:20 smithi006 ceph-mon[63965]: from='osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-17T10:55:20.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 217783 9.83 MB/sec execute 432 sec latency 109.277 ms 2024-09-17T10:55:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:20 smithi149 ceph-mon[49697]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.6 MiB/s wr, 327 op/s; 222/1671 objects degraded (13.285%) 2024-09-17T10:55:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:20 smithi149 ceph-mon[49697]: from='osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-17T10:55:21.236 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 10:55:21 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[76002]: 2024-09-17T10:55:21.127+0000 7ff552f4e640 -1 osd.3 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:55:21.751 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:21 smithi006 ceph-mon[63965]: from='osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T10:55:21.752 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:21 smithi006 ceph-mon[63965]: osdmap e109: 8 total, 7 up, 8 in 2024-09-17T10:55:21.752 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:21 smithi006 ceph-mon[63965]: from='osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]} : dispatch 2024-09-17T10:55:21.752 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:21 smithi006 ceph-mon[63965]: from='osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164]' entity='osd.3' 2024-09-17T10:55:21.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 218043 9.82 MB/sec execute 433 sec latency 123.261 ms 2024-09-17T10:55:21.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:21 smithi149 ceph-mon[49697]: from='osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T10:55:21.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:21 smithi149 ceph-mon[49697]: osdmap e109: 8 total, 7 up, 8 in 2024-09-17T10:55:21.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:21 smithi149 ceph-mon[49697]: from='osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]} : dispatch 2024-09-17T10:55:21.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:21 smithi149 ceph-mon[49697]: from='osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164]' entity='osd.3' 2024-09-17T10:55:22.769 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:22 smithi006 ceph-mon[63965]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.6 MiB/s wr, 339 op/s; 210/1587 objects degraded (13.233%) 2024-09-17T10:55:22.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:22 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 210/1587 objects degraded (13.233%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:22.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:22 smithi006 ceph-mon[63965]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:55:22.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:22 smithi006 ceph-mon[63965]: osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164] boot 2024-09-17T10:55:22.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:22 smithi006 ceph-mon[63965]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T10:55:22.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T10:55:22.770 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:22 smithi006 ceph-mon[63965]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T10:55:22.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 218292 9.82 MB/sec execute 434 sec latency 662.011 ms 2024-09-17T10:55:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:22 smithi149 ceph-mon[49697]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.6 MiB/s wr, 339 op/s; 210/1587 objects degraded (13.233%) 2024-09-17T10:55:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:22 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 210/1587 objects degraded (13.233%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:22 smithi149 ceph-mon[49697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:55:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:22 smithi149 ceph-mon[49697]: osd.3 [v2:172.21.15.6:6826/1528632164,v1:172.21.15.6:6827/1528632164] boot 2024-09-17T10:55:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:22 smithi149 ceph-mon[49697]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T10:55:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T10:55:22.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:22 smithi149 ceph-mon[49697]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T10:55:23.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 218494 9.81 MB/sec execute 435 sec latency 1114.902 ms 2024-09-17T10:55:24.323 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:24.323 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:24.323 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:55:24.323 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:55:24.323 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: pgmap v175: 97 pgs: 3 remapped+peering, 11 active+undersized, 20 active+undersized+degraded, 63 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 595 op/s; 149/1563 objects degraded (9.533%); 325 KiB/s, 4 objects/s recovering 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T10:55:24.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:24 smithi006 ceph-mon[63965]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-17T10:55:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:55:24.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: pgmap v175: 97 pgs: 3 remapped+peering, 11 active+undersized, 20 active+undersized+degraded, 63 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 595 op/s; 149/1563 objects degraded (9.533%); 325 KiB/s, 4 objects/s recovering 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T10:55:24.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:24 smithi149 ceph-mon[49697]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-17T10:55:24.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 218736 9.80 MB/sec execute 436 sec latency 68.067 ms 2024-09-17T10:55:25.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:25 smithi006 ceph-mon[63965]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T10:55:25.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:25 smithi149 ceph-mon[49697]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T10:55:25.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 219130 9.81 MB/sec execute 437 sec latency 348.135 ms 2024-09-17T10:55:26.375 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:26 smithi006 ceph-mon[63965]: osdmap e114: 8 total, 8 up, 8 in 2024-09-17T10:55:26.376 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:26 smithi006 ceph-mon[63965]: pgmap v178: 97 pgs: 3 remapped+peering, 6 active+undersized, 16 active+undersized+degraded, 72 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.0 MiB/s wr, 342 op/s; 114/1614 objects degraded (7.063%); 381 KiB/s, 17 objects/s recovering 2024-09-17T10:55:26.607 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:26 smithi149 ceph-mon[49697]: osdmap e114: 8 total, 8 up, 8 in 2024-09-17T10:55:26.607 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:26 smithi149 ceph-mon[49697]: pgmap v178: 97 pgs: 3 remapped+peering, 6 active+undersized, 16 active+undersized+degraded, 72 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.0 MiB/s wr, 342 op/s; 114/1614 objects degraded (7.063%); 381 KiB/s, 17 objects/s recovering 2024-09-17T10:55:26.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 219529 9.82 MB/sec execute 438 sec latency 667.742 ms 2024-09-17T10:55:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:27 smithi149 ceph-mon[49697]: osdmap e115: 8 total, 8 up, 8 in 2024-09-17T10:55:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:27 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 114/1614 objects degraded (7.063%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:27.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:27 smithi006 ceph-mon[63965]: osdmap e115: 8 total, 8 up, 8 in 2024-09-17T10:55:27.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:27 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 114/1614 objects degraded (7.063%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:27.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 219819 9.81 MB/sec execute 439 sec latency 83.478 ms 2024-09-17T10:55:28.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:28 smithi149 ceph-mon[49697]: pgmap v180: 97 pgs: 1 peering, 2 remapped+peering, 94 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s wr, 209 op/s; 1019 KiB/s, 48 objects/s recovering 2024-09-17T10:55:28.502 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:28 smithi006 ceph-mon[63965]: pgmap v180: 97 pgs: 1 peering, 2 remapped+peering, 94 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s wr, 209 op/s; 1019 KiB/s, 48 objects/s recovering 2024-09-17T10:55:28.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 220023 9.80 MB/sec execute 440 sec latency 68.320 ms 2024-09-17T10:55:29.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:29 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 114/1614 objects degraded (7.063%), 16 pgs degraded) 2024-09-17T10:55:29.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:29 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:55:29.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:29 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 114/1614 objects degraded (7.063%), 16 pgs degraded) 2024-09-17T10:55:29.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:29 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:55:29.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 220226 9.80 MB/sec execute 441 sec latency 57.762 ms 2024-09-17T10:55:30.366 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:30 smithi006 ceph-mon[63965]: pgmap v181: 97 pgs: 1 peering, 96 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.1 MiB/s wr, 224 op/s; 890 KiB/s, 41 objects/s recovering 2024-09-17T10:55:30.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:30.367 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:55:30.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:30 smithi149 ceph-mon[49697]: pgmap v181: 97 pgs: 1 peering, 96 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.1 MiB/s wr, 224 op/s; 890 KiB/s, 41 objects/s recovering 2024-09-17T10:55:30.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:30.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:55:30.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 220459 9.79 MB/sec execute 442 sec latency 85.273 ms 2024-09-17T10:55:31.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 220697 9.77 MB/sec execute 443 sec latency 90.156 ms 2024-09-17T10:55:32.704 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:32 smithi006 ceph-mon[63965]: pgmap v182: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.8 MiB/s wr, 430 op/s; 735 KiB/s, 33 objects/s recovering 2024-09-17T10:55:32.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 220996 9.77 MB/sec execute 444 sec latency 55.860 ms 2024-09-17T10:55:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:32 smithi149 ceph-mon[49697]: pgmap v182: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.8 MiB/s wr, 430 op/s; 735 KiB/s, 33 objects/s recovering 2024-09-17T10:55:33.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 221518 9.79 MB/sec execute 445 sec latency 34.435 ms 2024-09-17T10:55:34.735 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:34 smithi006 ceph-mon[63965]: pgmap v183: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 401 op/s; 647 KiB/s, 29 objects/s recovering 2024-09-17T10:55:34.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 222077 9.79 MB/sec execute 446 sec latency 27.360 ms 2024-09-17T10:55:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:34 smithi149 ceph-mon[49697]: pgmap v183: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 401 op/s; 647 KiB/s, 29 objects/s recovering 2024-09-17T10:55:35.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 222676 9.80 MB/sec execute 447 sec latency 27.493 ms 2024-09-17T10:55:36.391 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:55:36.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:36 smithi006 ceph-mon[63965]: pgmap v184: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 534 KiB/s, 20 objects/s recovering 2024-09-17T10:55:36.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 223247 9.82 MB/sec execute 448 sec latency 45.698 ms 2024-09-17T10:55:36.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:36 smithi149 ceph-mon[49697]: pgmap v184: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 534 KiB/s, 20 objects/s recovering 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (6m) 21s ago 16m 18.0M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (3m) 21s ago 15m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (3m) 3m ago 15m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (6m) 21s ago 15m 42.7M - dad864ee21e9 89df198e1a9b 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (10m) 21s ago 10m 38.0M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (9m) 3m ago 9m 14.7M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (5m) 21s ago 17m 518M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:55:37.560 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (5m) 3m ago 15m 453M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (4m) 21s ago 17m 52.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (4m) 3m ago 15m 35.2M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (9m) 21s ago 9m 286M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (9m) 3m ago 9m 86.0M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (6m) 21s ago 15m 9382k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (6m) 3m ago 15m 11.5M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (2m) 21s ago 14m 334M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (2m) 21s ago 13m 254M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (76s) 21s ago 13m 213M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (29s) 21s ago 13m 11.7M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (12m) 3m ago 12m 513M 4096M 17.2.0 e1d6a67b021e 97403b304e96 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (12m) 3m ago 12m 334M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (12m) 3m ago 12m 530M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:55:37.561 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (11m) 3m ago 11m 391M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:55:37.562 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (5m) 21s ago 14m 53.3M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:55:37.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 223621 9.83 MB/sec execute 449 sec latency 63.761 ms 2024-09-17T10:55:37.834 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:37 smithi006 ceph-mon[63965]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:37.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:37 smithi149 ceph-mon[49697]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:38.293 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:55:38.294 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:55:38.294 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:55:38.294 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:55:38.294 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:55:38.294 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T10:55:38.295 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T10:55:38.296 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:55:38.296 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:55:38.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 223895 9.82 MB/sec execute 450 sec latency 51.448 ms 2024-09-17T10:55:38.841 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:55:38.842 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:55:38.842 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:55:38.842 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:55:38.842 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:55:38.842 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:55:38.842 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:55:38.843 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:55:38.843 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:55:38.843 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "10/23 daemons upgraded", 2024-09-17T10:55:38.843 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:55:38.843 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:55:38.843 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:55:38.844 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:38 smithi006 ceph-mon[63965]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:38.845 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:38 smithi006 ceph-mon[63965]: pgmap v185: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.8 MiB/s wr, 485 op/s; 90 KiB/s, 5 objects/s recovering 2024-09-17T10:55:38.845 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:38 smithi006 ceph-mon[63965]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:38.845 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:38 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2177158770' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:38 smithi149 ceph-mon[49697]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:38 smithi149 ceph-mon[49697]: pgmap v185: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.8 MiB/s wr, 485 op/s; 90 KiB/s, 5 objects/s recovering 2024-09-17T10:55:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:38 smithi149 ceph-mon[49697]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:38 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2177158770' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:55:39.478 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:55:39.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:39 smithi006 ceph-mon[63965]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:39.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T10:55:39.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:39 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T10:55:39.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:39 smithi006 ceph-mon[63965]: Upgrade: osd.4 is safe to restart 2024-09-17T10:55:39.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:39 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2771498166' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:55:39.833 INFO:teuthology.orchestra.run.smithi006.stdout: 5 224189 9.81 MB/sec execute 451 sec latency 89.643 ms 2024-09-17T10:55:39.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:39 smithi149 ceph-mon[49697]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:55:39.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T10:55:39.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:39 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T10:55:39.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:39 smithi149 ceph-mon[49697]: Upgrade: osd.4 is safe to restart 2024-09-17T10:55:39.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:39 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2771498166' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:55:40.710 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:40 smithi006 ceph-mon[63965]: pgmap v186: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 82 KiB/s, 3 objects/s recovering 2024-09-17T10:55:40.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:40.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:40.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T10:55:40.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:55:40.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 224368 9.80 MB/sec execute 452 sec latency 61.469 ms 2024-09-17T10:55:40.935 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:40 smithi149 ceph-mon[49697]: pgmap v186: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 82 KiB/s, 3 objects/s recovering 2024-09-17T10:55:40.935 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:40.935 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:40.935 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T10:55:40.935 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:55:41.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:41 smithi006 ceph-mon[63965]: Upgrade: Updating osd.4 2024-09-17T10:55:41.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:41 smithi006 ceph-mon[63965]: Deploying daemon osd.4 on smithi149 2024-09-17T10:55:41.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 224577 9.79 MB/sec execute 453 sec latency 74.190 ms 2024-09-17T10:55:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:41 smithi149 ceph-mon[49697]: Upgrade: Updating osd.4 2024-09-17T10:55:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:41 smithi149 ceph-mon[49697]: Deploying daemon osd.4 on smithi149 2024-09-17T10:55:42.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:42 smithi006 ceph-mon[63965]: pgmap v187: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.7 MiB/s wr, 549 op/s 2024-09-17T10:55:42.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 224803 9.79 MB/sec execute 454 sec latency 93.991 ms 2024-09-17T10:55:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:42 smithi149 ceph-mon[49697]: pgmap v187: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.7 MiB/s wr, 549 op/s 2024-09-17T10:55:43.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 224968 9.78 MB/sec execute 455 sec latency 200.848 ms 2024-09-17T10:55:44.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:44 smithi006 ceph-mon[63965]: pgmap v188: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.1 MiB/s wr, 496 op/s 2024-09-17T10:55:44.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:44 smithi149 ceph-mon[49697]: pgmap v188: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.1 MiB/s wr, 496 op/s 2024-09-17T10:55:44.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 225176 9.76 MB/sec execute 456 sec latency 86.270 ms 2024-09-17T10:55:45.206 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:44 smithi149 systemd[1]: Stopping Ceph osd.4 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:55:45.638 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:45 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[30273]: 2024-09-17T10:55:45.263+0000 7f1412beb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:55:45.639 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:45 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[30273]: 2024-09-17T10:55:45.263+0000 7f1412beb700 -1 osd.4 115 *** Got signal Terminated *** 2024-09-17T10:55:45.639 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:45 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[30273]: 2024-09-17T10:55:45.263+0000 7f1412beb700 -1 osd.4 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T10:55:45.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:55:45.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:45 smithi006 ceph-mon[63965]: osd.4 marked itself down and dead 2024-09-17T10:55:45.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 225512 9.76 MB/sec execute 457 sec latency 173.151 ms 2024-09-17T10:55:45.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:55:45.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:45 smithi149 ceph-mon[49697]: osd.4 marked itself down and dead 2024-09-17T10:55:46.532 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:46 smithi149 podman[53102]: 2024-09-17 10:55:46.249768061 +0000 UTC m=+1.157150628 container died 97403b304e96939a5e8ef64d5c6af8f0a202ea1cd36bcd2a04da389e2bf308f6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , release=754, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream) 2024-09-17T10:55:46.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 225594 9.74 MB/sec execute 458 sec latency 1024.463 ms 2024-09-17T10:55:46.888 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:46 smithi006 ceph-mon[63965]: pgmap v189: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.4 MiB/s wr, 577 op/s 2024-09-17T10:55:46.888 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:46 smithi006 ceph-mon[63965]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:55:46.888 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:46 smithi006 ceph-mon[63965]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T10:55:46.932 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:46 smithi149 ceph-mon[49697]: pgmap v189: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.4 MiB/s wr, 577 op/s 2024-09-17T10:55:46.932 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:46 smithi149 ceph-mon[49697]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:55:46.932 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:46 smithi149 ceph-mon[49697]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T10:55:46.932 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:46 smithi149 podman[53102]: 2024-09-17 10:55:46.620970429 +0000 UTC m=+1.528352989 container cleanup 97403b304e96939a5e8ef64d5c6af8f0a202ea1cd36bcd2a04da389e2bf308f6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, version=8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-17T10:55:46.932 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:46 smithi149 bash[53102]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4 2024-09-17T10:55:47.206 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:46 smithi149 podman[53119]: 2024-09-17 10:55:46.931107073 +0000 UTC m=+0.672919536 container remove 97403b304e96939a5e8ef64d5c6af8f0a202ea1cd36bcd2a04da389e2bf308f6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, distribution-scope=public, release=754, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, ceph=True, maintainer=Guillaume Abrioux , architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=) 2024-09-17T10:55:47.562 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:47 smithi149 podman[53208]: 2024-09-17 10:55:47.196511737 +0000 UTC m=+0.018197993 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:55:47.562 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:47 smithi149 podman[53208]: 2024-09-17 10:55:47.315704129 +0000 UTC m=+0.137390384 container create 1d933e8915146253de99089512e787f823eaa8b4f130bc5d0ff00b0e68132279 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-7498493, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T10:55:47.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:47 smithi006 ceph-mon[63965]: osdmap e117: 8 total, 7 up, 8 in 2024-09-17T10:55:47.823 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:47 smithi149 ceph-mon[49697]: osdmap e117: 8 total, 7 up, 8 in 2024-09-17T10:55:47.823 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:47 smithi149 podman[53208]: 2024-09-17 10:55:47.56131004 +0000 UTC m=+0.382996296 container init 1d933e8915146253de99089512e787f823eaa8b4f130bc5d0ff00b0e68132279 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T10:55:47.823 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:47 smithi149 podman[53208]: 2024-09-17 10:55:47.566836541 +0000 UTC m=+0.388522806 container start 1d933e8915146253de99089512e787f823eaa8b4f130bc5d0ff00b0e68132279 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:55:47.823 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:47 smithi149 podman[53208]: 2024-09-17 10:55:47.661282348 +0000 UTC m=+0.482968604 container attach 1d933e8915146253de99089512e787f823eaa8b4f130bc5d0ff00b0e68132279 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-17T10:55:47.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 225819 9.73 MB/sec execute 459 sec latency 47.200 ms 2024-09-17T10:55:48.184 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:47 smithi149 podman[53208]: 2024-09-17 10:55:47.822148364 +0000 UTC m=+0.643834628 container died 1d933e8915146253de99089512e787f823eaa8b4f130bc5d0ff00b0e68132279 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-17T10:55:48.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:48 smithi006 ceph-mon[63965]: pgmap v192: 97 pgs: 3 active+undersized, 5 peering, 9 stale+active+clean, 4 active+undersized+degraded, 76 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 501 op/s; 33/1578 objects degraded (2.091%) 2024-09-17T10:55:48.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:48 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 33/1578 objects degraded (2.091%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:48.720 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:48 smithi149 podman[53232]: 2024-09-17 10:55:48.467020896 +0000 UTC m=+0.638203056 container remove 1d933e8915146253de99089512e787f823eaa8b4f130bc5d0ff00b0e68132279 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-17T10:55:48.721 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:48 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T10:55:48.721 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:48 smithi149 systemd[1]: Stopped Ceph osd.4 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:55:48.721 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:48 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.4.service: Consumed 3min 59.826s CPU time. 2024-09-17T10:55:48.721 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:48 smithi149 ceph-mon[49697]: pgmap v192: 97 pgs: 3 active+undersized, 5 peering, 9 stale+active+clean, 4 active+undersized+degraded, 76 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 501 op/s; 33/1578 objects degraded (2.091%) 2024-09-17T10:55:48.721 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:48 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 33/1578 objects degraded (2.091%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:48.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 226108 9.73 MB/sec execute 460 sec latency 50.025 ms 2024-09-17T10:55:48.986 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:48 smithi149 systemd[1]: Starting Ceph osd.4 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:55:48.986 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:48 smithi149 podman[53328]: 2024-09-17 10:55:48.868687997 +0000 UTC m=+0.017603431 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:55:49.581 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:48 smithi149 podman[53328]: 2024-09-17 10:55:48.985632191 +0000 UTC m=+0.134547612 container create 668f8f0d97d48df40b0140c8b9f3660c616cea08a61a4aecfe2892bcc7f5ec64 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, RELEASE=main-7498493, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:55:49.582 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:49 smithi149 podman[53328]: 2024-09-17 10:55:49.422587732 +0000 UTC m=+0.571503153 container init 668f8f0d97d48df40b0140c8b9f3660c616cea08a61a4aecfe2892bcc7f5ec64 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T10:55:49.582 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:49 smithi149 podman[53328]: 2024-09-17 10:55:49.426624721 +0000 UTC m=+0.575540155 container start 668f8f0d97d48df40b0140c8b9f3660c616cea08a61a4aecfe2892bcc7f5ec64 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:55:49.582 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:49 smithi149 podman[53328]: 2024-09-17 10:55:49.506025607 +0000 UTC m=+0.654941037 container attach 668f8f0d97d48df40b0140c8b9f3660c616cea08a61a4aecfe2892bcc7f5ec64 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T10:55:49.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 226334 9.72 MB/sec execute 461 sec latency 55.736 ms 2024-09-17T10:55:49.956 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:55:49.956 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:49 smithi149 bash[53328]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:55:49.956 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:55:49.956 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:49 smithi149 bash[53328]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:55:50.763 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:50 smithi006 ceph-mon[63965]: pgmap v193: 97 pgs: 5 active+undersized, 5 peering, 5 stale+active+clean, 12 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.0 MiB/s wr, 349 op/s; 113/1584 objects degraded (7.134%) 2024-09-17T10:55:50.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 226637 9.71 MB/sec execute 462 sec latency 77.455 ms 2024-09-17T10:55:50.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:50 smithi149 ceph-mon[49697]: pgmap v193: 97 pgs: 5 active+undersized, 5 peering, 5 stale+active+clean, 12 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.0 MiB/s wr, 349 op/s; 113/1584 objects degraded (7.134%) 2024-09-17T10:55:51.277 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:50 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T10:55:51.278 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:50 smithi149 bash[53328]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T10:55:51.278 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:50 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T10:55:51.278 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:50 smithi149 bash[53328]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T10:55:51.278 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T10:55:51.278 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 bash[53328]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T10:55:51.661 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T10:55:51.661 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 bash[53328]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T10:55:51.661 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T10:55:51.661 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 bash[53328]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T10:55:51.661 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T10:55:51.662 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 bash[53328]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T10:55:51.662 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate[53343]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T10:55:51.662 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 bash[53328]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T10:55:51.662 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 podman[53328]: 2024-09-17 10:55:51.306828807 +0000 UTC m=+2.455744235 container died 668f8f0d97d48df40b0140c8b9f3660c616cea08a61a4aecfe2892bcc7f5ec64 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-7498493, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:55:51.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 226985 9.72 MB/sec execute 463 sec latency 37.559 ms 2024-09-17T10:55:52.330 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:51 smithi149 podman[53497]: 2024-09-17 10:55:51.960842065 +0000 UTC m=+0.648103324 container remove 668f8f0d97d48df40b0140c8b9f3660c616cea08a61a4aecfe2892bcc7f5ec64 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-7498493, maintainer=Guillaume Abrioux ) 2024-09-17T10:55:52.331 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:52 smithi149 podman[53556]: 2024-09-17 10:55:52.073968229 +0000 UTC m=+0.017274519 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:55:52.331 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:52 smithi149 podman[53556]: 2024-09-17 10:55:52.203741373 +0000 UTC m=+0.147047655 container create 016956823ff1173cf85e4c4cb00affa5c291b010d4737596f47c4f55c25697df (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-17T10:55:52.667 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:52 smithi149 podman[53556]: 2024-09-17 10:55:52.464895389 +0000 UTC m=+0.408201680 container init 016956823ff1173cf85e4c4cb00affa5c291b010d4737596f47c4f55c25697df (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:55:52.667 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:52 smithi149 podman[53556]: 2024-09-17 10:55:52.46824338 +0000 UTC m=+0.411549663 container start 016956823ff1173cf85e4c4cb00affa5c291b010d4737596f47c4f55c25697df (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T10:55:52.667 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:52 smithi149 ceph-osd[53575]: -- 172.21.15.149:0/585328939 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5643b68d61e0 con 0x5643b68d2000 2024-09-17T10:55:52.667 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:52 smithi149 bash[53556]: 016956823ff1173cf85e4c4cb00affa5c291b010d4737596f47c4f55c25697df 2024-09-17T10:55:52.667 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:52 smithi149 systemd[1]: Started Ceph osd.4 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:55:52.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:52 smithi006 ceph-mon[63965]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.3 MiB/s wr, 378 op/s; 252/1599 objects degraded (15.760%) 2024-09-17T10:55:52.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:52.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:52.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:55:52.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 227241 9.72 MB/sec execute 464 sec latency 39.206 ms 2024-09-17T10:55:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:52 smithi149 ceph-mon[49697]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.3 MiB/s wr, 378 op/s; 252/1599 objects degraded (15.760%) 2024-09-17T10:55:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:55:53.822 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:53 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 254/1614 objects degraded (15.737%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:53.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 227559 9.72 MB/sec execute 465 sec latency 67.597 ms 2024-09-17T10:55:54.058 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:55:53 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T10:55:53.871+0000 7ff72f96d640 -1 Falling back to public interface 2024-09-17T10:55:54.058 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:53 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 254/1614 objects degraded (15.737%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:54.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 227861 9.71 MB/sec execute 466 sec latency 48.161 ms 2024-09-17T10:55:54.836 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:54 smithi006 ceph-mon[63965]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 378 op/s; 254/1614 objects degraded (15.737%) 2024-09-17T10:55:55.041 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:54 smithi149 ceph-mon[49697]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 378 op/s; 254/1614 objects degraded (15.737%) 2024-09-17T10:55:55.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 228079 9.70 MB/sec execute 467 sec latency 84.616 ms 2024-09-17T10:55:56.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 228369 9.70 MB/sec execute 468 sec latency 57.386 ms 2024-09-17T10:55:56.847 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:56 smithi149 ceph-mon[49697]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.1 MiB/s wr, 305 op/s; 259/1635 objects degraded (15.841%) 2024-09-17T10:55:56.847 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:56.847 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:56.876 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:56 smithi006 ceph-mon[63965]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.1 MiB/s wr, 305 op/s; 259/1635 objects degraded (15.841%) 2024-09-17T10:55:56.877 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:56.877 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:57.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 228636 9.69 MB/sec execute 469 sec latency 55.562 ms 2024-09-17T10:55:58.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:58 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:58.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:58 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:58.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:58 smithi006 ceph-mon[63965]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.9 MiB/s wr, 376 op/s; 265/1695 objects degraded (15.634%) 2024-09-17T10:55:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:58 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:58 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:55:58.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:58 smithi149 ceph-mon[49697]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.9 MiB/s wr, 376 op/s; 265/1695 objects degraded (15.634%) 2024-09-17T10:55:58.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 229044 9.69 MB/sec execute 470 sec latency 42.466 ms 2024-09-17T10:55:59.714 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:55:59 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 263/1674 objects degraded (15.711%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T10:55:59.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 229268 9.68 MB/sec execute 471 sec latency 56.810 ms 2024-09-17T10:55:59.888 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:55:59 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 263/1674 objects degraded (15.711%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:00.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 229596 9.68 MB/sec execute 472 sec latency 97.059 ms 2024-09-17T10:56:00.999 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T10:56:00.725+0000 7ff72f96d640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 370 op/s; 263/1674 objects degraded (15.711%) 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 370 op/s; 263/1674 objects degraded (15.711%) 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:01.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 229929 9.68 MB/sec execute 473 sec latency 41.496 ms 2024-09-17T10:56:02.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:01 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:56:02.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:01 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:02.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:01 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:02.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:01 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:02.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:02 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T10:56:02.456 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:56:02 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T10:56:02.053+0000 7ff72f96d640 -1 osd.4 115 log_to_monitors true 2024-09-17T10:56:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:56:02.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:02.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:02 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T10:56:02.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 230352 9.68 MB/sec execute 474 sec latency 71.277 ms 2024-09-17T10:56:03.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:03 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T10:56:03.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:03 smithi006 ceph-mon[63965]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T10:56:03.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:03 smithi006 ceph-mon[63965]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 403 op/s; 263/1656 objects degraded (15.882%) 2024-09-17T10:56:03.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:03 smithi006 ceph-mon[63965]: from='osd.4 [v2:172.21.15.149:6800/3742471406,v1:172.21.15.149:6801/3742471406]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T10:56:03.052 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:03 smithi006 ceph-mon[63965]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T10:56:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:03 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T10:56:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:03 smithi149 ceph-mon[49697]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T10:56:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:03 smithi149 ceph-mon[49697]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 403 op/s; 263/1656 objects degraded (15.882%) 2024-09-17T10:56:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:03 smithi149 ceph-mon[49697]: from='osd.4 [v2:172.21.15.149:6800/3742471406,v1:172.21.15.149:6801/3742471406]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T10:56:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:03 smithi149 ceph-mon[49697]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T10:56:03.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 230560 9.68 MB/sec execute 475 sec latency 145.831 ms 2024-09-17T10:56:03.956 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 10:56:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T10:56:03.486+0000 7ff72370f640 -1 osd.4 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:56:04.133 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:04 smithi006 ceph-mon[63965]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T10:56:04.134 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:04 smithi006 ceph-mon[63965]: from='osd.4 [v2:172.21.15.149:6800/3742471406,v1:172.21.15.149:6801/3742471406]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:56:04.134 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:04 smithi006 ceph-mon[63965]: osdmap e118: 8 total, 7 up, 8 in 2024-09-17T10:56:04.134 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:04 smithi006 ceph-mon[63965]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:56:04.134 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:04 smithi006 ceph-mon[63965]: from='osd.4 ' entity='osd.4' 2024-09-17T10:56:04.134 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:04 smithi006 ceph-mon[63965]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 393 op/s; 266/1695 objects degraded (15.693%) 2024-09-17T10:56:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:04 smithi149 ceph-mon[49697]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T10:56:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:04 smithi149 ceph-mon[49697]: from='osd.4 [v2:172.21.15.149:6800/3742471406,v1:172.21.15.149:6801/3742471406]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:56:04.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:04 smithi149 ceph-mon[49697]: osdmap e118: 8 total, 7 up, 8 in 2024-09-17T10:56:04.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:04 smithi149 ceph-mon[49697]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:56:04.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:04 smithi149 ceph-mon[49697]: from='osd.4 ' entity='osd.4' 2024-09-17T10:56:04.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:04 smithi149 ceph-mon[49697]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 393 op/s; 266/1695 objects degraded (15.693%) 2024-09-17T10:56:04.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 230817 9.67 MB/sec execute 476 sec latency 277.230 ms 2024-09-17T10:56:05.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:05 smithi006 ceph-mon[63965]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:56:05.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:05 smithi006 ceph-mon[63965]: osd.4 [v2:172.21.15.149:6800/3742471406,v1:172.21.15.149:6801/3742471406] boot 2024-09-17T10:56:05.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:05 smithi006 ceph-mon[63965]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T10:56:05.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:05 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T10:56:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:05 smithi149 ceph-mon[49697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:56:05.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:05 smithi149 ceph-mon[49697]: osd.4 [v2:172.21.15.149:6800/3742471406,v1:172.21.15.149:6801/3742471406] boot 2024-09-17T10:56:05.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:05 smithi149 ceph-mon[49697]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T10:56:05.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:05 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T10:56:05.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 230947 9.66 MB/sec execute 477 sec latency 1218.973 ms 2024-09-17T10:56:06.567 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:06 smithi006 ceph-mon[63965]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T10:56:06.601 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:06 smithi006 ceph-mon[63965]: pgmap v204: 97 pgs: 7 peering, 13 active+undersized, 21 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 555 op/s; 224/1689 objects degraded (13.262%) 2024-09-17T10:56:06.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 231200 9.66 MB/sec execute 478 sec latency 1924.809 ms 2024-09-17T10:56:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:06 smithi149 ceph-mon[49697]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T10:56:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:06 smithi149 ceph-mon[49697]: pgmap v204: 97 pgs: 7 peering, 13 active+undersized, 21 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 555 op/s; 224/1689 objects degraded (13.262%) 2024-09-17T10:56:07.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:07 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 224/1689 objects degraded (13.262%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:07.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:07 smithi006 ceph-mon[63965]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T10:56:07.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:07 smithi006 ceph-mon[63965]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T10:56:07.820 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:07 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 224/1689 objects degraded (13.262%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:07.820 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:07 smithi149 ceph-mon[49697]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T10:56:07.820 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:07 smithi149 ceph-mon[49697]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T10:56:07.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 231521 9.66 MB/sec execute 479 sec latency 68.610 ms 2024-09-17T10:56:08.595 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:08 smithi006 ceph-mon[63965]: pgmap v207: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 remapped+peering, 7 peering, 3 active+recovery_wait+degraded, 10 active+undersized, 9 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.9 MiB/s wr, 383 op/s; 161/1647 objects degraded (9.775%); 1.0 MiB/s, 57 keys/s, 2 objects/s recovering 2024-09-17T10:56:08.596 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:08 smithi006 ceph-mon[63965]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T10:56:08.834 INFO:teuthology.orchestra.run.smithi006.stdout: 5 231700 9.64 MB/sec execute 480 sec latency 81.222 ms 2024-09-17T10:56:08.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:08 smithi149 ceph-mon[49697]: pgmap v207: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 remapped+peering, 7 peering, 3 active+recovery_wait+degraded, 10 active+undersized, 9 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.9 MiB/s wr, 383 op/s; 161/1647 objects degraded (9.775%); 1.0 MiB/s, 57 keys/s, 2 objects/s recovering 2024-09-17T10:56:08.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:08 smithi149 ceph-mon[49697]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T10:56:09.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 231975 9.64 MB/sec execute 481 sec latency 301.044 ms 2024-09-17T10:56:10.004 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:56:10.376 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:10 smithi006 ceph-mon[63965]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T10:56:10.376 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:10 smithi006 ceph-mon[63965]: pgmap v210: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 remapped+peering, 7 peering, 4 active+recovery_wait+degraded, 79 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.2 MiB/s wr, 145 op/s; 109/1650 objects degraded (6.606%); 1.3 MiB/s, 57 keys/s, 20 objects/s recovering 2024-09-17T10:56:10.376 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:10 smithi006 ceph-mon[63965]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:10.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:10 smithi149 ceph-mon[49697]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T10:56:10.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:10 smithi149 ceph-mon[49697]: pgmap v210: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 remapped+peering, 7 peering, 4 active+recovery_wait+degraded, 79 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.2 MiB/s wr, 145 op/s; 109/1650 objects degraded (6.606%); 1.3 MiB/s, 57 keys/s, 20 objects/s recovering 2024-09-17T10:56:10.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:10 smithi149 ceph-mon[49697]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:10.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 232225 9.62 MB/sec execute 482 sec latency 740.923 ms 2024-09-17T10:56:10.996 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:56:10.996 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (7m) 55s ago 16m 18.0M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:56:10.996 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (4m) 55s ago 16m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:56:10.996 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (3m) 14s ago 15m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:56:10.996 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (7m) 55s ago 16m 42.7M - dad864ee21e9 89df198e1a9b 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (10m) 55s ago 10m 38.0M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (10m) 14s ago 10m 15.7M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (6m) 55s ago 17m 518M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (5m) 14s ago 15m 456M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (5m) 55s ago 17m 52.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (4m) 14s ago 15m 40.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (10m) 55s ago 10m 286M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (10m) 14s ago 10m 88.5M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (7m) 55s ago 16m 9382k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (6m) 14s ago 15m 10.8M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (3m) 55s ago 15m 334M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (2m) 55s ago 14m 254M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (110s) 55s ago 14m 213M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:56:10.997 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (62s) 55s ago 13m 11.7M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T10:56:10.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (18s) 14s ago 13m 11.2M 4096M 19.3.0-4937-g74984932 6c1fcab85602 016956823ff1 2024-09-17T10:56:10.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (12m) 14s ago 12m 541M 4096M 17.2.0 e1d6a67b021e 313d2d1411b4 2024-09-17T10:56:10.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (12m) 14s ago 12m 462M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:56:10.998 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (12m) 14s ago 12m 664M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:56:10.998 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (5m) 55s ago 15m 53.3M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:56:11.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:11 smithi006 ceph-mon[63965]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T10:56:11.400 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:11 smithi006 ceph-mon[63965]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:11.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:11 smithi006 ceph-mon[63965]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:11.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:11 smithi006 ceph-mon[63965]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T10:56:11.563 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:56:11.563 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:56:11.563 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:56:11.563 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:56:11.563 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:56:11.563 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:56:11.563 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 5 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 9 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:56:11.564 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:56:11.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:11 smithi149 ceph-mon[49697]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T10:56:11.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:11 smithi149 ceph-mon[49697]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:11.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:11 smithi149 ceph-mon[49697]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:11.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:11 smithi149 ceph-mon[49697]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T10:56:11.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 232510 9.62 MB/sec execute 483 sec latency 640.596 ms 2024-09-17T10:56:12.094 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "11/23 daemons upgraded", 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:56:12.095 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:56:12.405 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:12 smithi006 ceph-mon[63965]: pgmap v213: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 89 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.0 MiB/s wr, 461 op/s; 74/1665 objects degraded (4.444%); 255 KiB/s, 60 objects/s recovering 2024-09-17T10:56:12.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:12 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3909301603' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:12.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:12 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 109/1650 objects degraded (6.606%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:12.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:12 smithi006 ceph-mon[63965]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T10:56:12.694 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN Degraded data redundancy: 74/1665 objects degraded (4.444%), 4 pgs degraded 2024-09-17T10:56:12.694 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 74/1665 objects degraded (4.444%), 4 pgs degraded 2024-09-17T10:56:12.695 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2024-09-17T10:56:12.695 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.19 is active+recovery_wait+degraded, acting [1,3,4] 2024-09-17T10:56:12.695 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1a is active+recovery_wait+undersized+degraded+remapped, acting [1,2] 2024-09-17T10:56:12.695 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1b is active+recovery_wait+degraded, acting [0,4,7] 2024-09-17T10:56:12.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:12 smithi149 ceph-mon[49697]: pgmap v213: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 89 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.0 MiB/s wr, 461 op/s; 74/1665 objects degraded (4.444%); 255 KiB/s, 60 objects/s recovering 2024-09-17T10:56:12.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:12 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3909301603' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:12.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:12 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 109/1650 objects degraded (6.606%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:12.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:12 smithi149 ceph-mon[49697]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T10:56:12.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 232832 9.62 MB/sec execute 484 sec latency 533.539 ms 2024-09-17T10:56:13.448 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:13 smithi006 ceph-mon[63965]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:13.448 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:13 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3159983138' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:56:13.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:13 smithi149 ceph-mon[49697]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:13.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:13 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3159983138' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:56:13.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 233308 9.63 MB/sec execute 485 sec latency 37.532 ms 2024-09-17T10:56:14.465 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:14 smithi006 ceph-mon[63965]: pgmap v215: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.6 MiB/s wr, 456 op/s; 509 KiB/s, 57 objects/s recovering 2024-09-17T10:56:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:14 smithi149 ceph-mon[49697]: pgmap v215: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.6 MiB/s wr, 456 op/s; 509 KiB/s, 57 objects/s recovering 2024-09-17T10:56:14.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 233607 9.62 MB/sec execute 486 sec latency 32.590 ms 2024-09-17T10:56:15.482 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:15 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/1665 objects degraded (4.444%), 4 pgs degraded) 2024-09-17T10:56:15.483 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:15 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:56:15.483 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:15.483 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:56:15.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:15 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/1665 objects degraded (4.444%), 4 pgs degraded) 2024-09-17T10:56:15.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:15 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:56:15.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:15.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:56:15.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 233955 9.62 MB/sec execute 487 sec latency 32.367 ms 2024-09-17T10:56:16.502 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:16 smithi006 ceph-mon[63965]: pgmap v216: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 345 op/s; 363 KiB/s, 40 objects/s recovering 2024-09-17T10:56:16.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:16 smithi149 ceph-mon[49697]: pgmap v216: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 345 op/s; 363 KiB/s, 40 objects/s recovering 2024-09-17T10:56:16.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 234320 9.63 MB/sec execute 488 sec latency 40.491 ms 2024-09-17T10:56:17.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:17 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T10:56:17.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:17 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T10:56:17.408 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:17 smithi006 ceph-mon[63965]: Upgrade: osd.5 is safe to restart 2024-09-17T10:56:17.632 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:17 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T10:56:17.632 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:17 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T10:56:17.632 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:17 smithi149 ceph-mon[49697]: Upgrade: osd.5 is safe to restart 2024-09-17T10:56:17.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 234673 9.63 MB/sec execute 489 sec latency 43.898 ms 2024-09-17T10:56:18.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:18 smithi006 ceph-mon[63965]: pgmap v217: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.8 MiB/s wr, 491 op/s; 299 KiB/s, 33 objects/s recovering 2024-09-17T10:56:18.640 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:18 smithi149 ceph-mon[49697]: pgmap v217: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.8 MiB/s wr, 491 op/s; 299 KiB/s, 33 objects/s recovering 2024-09-17T10:56:18.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 234975 9.64 MB/sec execute 490 sec latency 38.818 ms 2024-09-17T10:56:19.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:19 smithi006 ceph-mon[63965]: Upgrade: Updating osd.5 2024-09-17T10:56:19.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:19.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T10:56:19.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:56:19.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:19 smithi006 ceph-mon[63965]: Deploying daemon osd.5 on smithi149 2024-09-17T10:56:19.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 235334 9.62 MB/sec execute 491 sec latency 40.365 ms 2024-09-17T10:56:19.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:19 smithi149 ceph-mon[49697]: Upgrade: Updating osd.5 2024-09-17T10:56:19.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:19.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T10:56:19.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:56:19.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:19 smithi149 ceph-mon[49697]: Deploying daemon osd.5 on smithi149 2024-09-17T10:56:20.715 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:20 smithi006 ceph-mon[63965]: pgmap v218: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 404 op/s; 261 KiB/s, 8 objects/s recovering 2024-09-17T10:56:20.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 235571 9.61 MB/sec execute 492 sec latency 49.544 ms 2024-09-17T10:56:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:20 smithi149 ceph-mon[49697]: pgmap v218: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 404 op/s; 261 KiB/s, 8 objects/s recovering 2024-09-17T10:56:21.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 235788 9.60 MB/sec execute 493 sec latency 62.615 ms 2024-09-17T10:56:22.037 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:21 smithi149 systemd[1]: Stopping Ceph osd.5 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:56:22.295 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:22 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[33217]: 2024-09-17T10:56:22.053+0000 7f610f6cf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:56:22.296 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:22 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[33217]: 2024-09-17T10:56:22.053+0000 7f610f6cf700 -1 osd.5 127 *** Got signal Terminated *** 2024-09-17T10:56:22.296 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:22 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[33217]: 2024-09-17T10:56:22.053+0000 7f610f6cf700 -1 osd.5 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T10:56:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:22 smithi006 ceph-mon[63965]: pgmap v219: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 399 op/s; 216 KiB/s, 7 objects/s recovering 2024-09-17T10:56:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:22 smithi006 ceph-mon[63965]: osd.5 marked itself down and dead 2024-09-17T10:56:22.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 236068 9.61 MB/sec execute 494 sec latency 68.248 ms 2024-09-17T10:56:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:22 smithi149 ceph-mon[49697]: pgmap v219: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 399 op/s; 216 KiB/s, 7 objects/s recovering 2024-09-17T10:56:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:22 smithi149 ceph-mon[49697]: osd.5 marked itself down and dead 2024-09-17T10:56:23.456 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:23 smithi149 podman[55134]: 2024-09-17 10:56:23.346972928 +0000 UTC m=+1.399535952 container died 313d2d1411b40616dd31b8ff60c366150a8ff9e997d268e3ef9807bc26db2db0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, distribution-scope=public, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.openshift.expose-services=, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T10:56:23.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 236204 9.60 MB/sec execute 495 sec latency 911.493 ms 2024-09-17T10:56:23.954 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:23 smithi006 ceph-mon[63965]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:56:23.955 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:23 smithi006 ceph-mon[63965]: osdmap e128: 8 total, 7 up, 8 in 2024-09-17T10:56:23.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:23 smithi149 podman[55134]: 2024-09-17 10:56:23.821044079 +0000 UTC m=+1.873607099 container cleanup 313d2d1411b40616dd31b8ff60c366150a8ff9e997d268e3ef9807bc26db2db0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, ceph=True, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-type=git, release=754, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., name=centos-stream, io.openshift.expose-services=, architecture=x86_64) 2024-09-17T10:56:23.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:23 smithi149 bash[55134]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5 2024-09-17T10:56:23.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:23 smithi149 ceph-mon[49697]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:56:23.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:23 smithi149 ceph-mon[49697]: osdmap e128: 8 total, 7 up, 8 in 2024-09-17T10:56:24.456 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:24 smithi149 podman[55161]: 2024-09-17 10:56:24.131301176 +0000 UTC m=+0.781300120 container remove 313d2d1411b40616dd31b8ff60c366150a8ff9e997d268e3ef9807bc26db2db0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8) 2024-09-17T10:56:24.735 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:24 smithi006 ceph-mon[63965]: pgmap v221: 97 pgs: 10 peering, 8 stale+active+clean, 79 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 493 op/s 2024-09-17T10:56:24.735 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:24 smithi006 ceph-mon[63965]: osdmap e129: 8 total, 7 up, 8 in 2024-09-17T10:56:24.821 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:24 smithi149 ceph-mon[49697]: pgmap v221: 97 pgs: 10 peering, 8 stale+active+clean, 79 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 493 op/s 2024-09-17T10:56:24.821 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:24 smithi149 ceph-mon[49697]: osdmap e129: 8 total, 7 up, 8 in 2024-09-17T10:56:24.822 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:24 smithi149 podman[55247]: 2024-09-17 10:56:24.393970392 +0000 UTC m=+0.018416370 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:56:24.822 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:24 smithi149 podman[55247]: 2024-09-17 10:56:24.501064118 +0000 UTC m=+0.125510092 container create 9161ab8a8bc64b0588869d92d85b914ec4a3bd3705a4161e848d92a6cde61c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493) 2024-09-17T10:56:24.822 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:24 smithi149 podman[55247]: 2024-09-17 10:56:24.720648977 +0000 UTC m=+0.345094951 container init 9161ab8a8bc64b0588869d92d85b914ec4a3bd3705a4161e848d92a6cde61c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-17T10:56:24.822 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:24 smithi149 podman[55247]: 2024-09-17 10:56:24.72545066 +0000 UTC m=+0.349896637 container start 9161ab8a8bc64b0588869d92d85b914ec4a3bd3705a4161e848d92a6cde61c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-17T10:56:24.822 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:24 smithi149 podman[55247]: 2024-09-17 10:56:24.820786875 +0000 UTC m=+0.445232859 container attach 9161ab8a8bc64b0588869d92d85b914ec4a3bd3705a4161e848d92a6cde61c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-17T10:56:24.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 236492 9.59 MB/sec execute 496 sec latency 72.411 ms 2024-09-17T10:56:25.206 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:24 smithi149 podman[55247]: 2024-09-17 10:56:24.957121007 +0000 UTC m=+0.581566987 container died 9161ab8a8bc64b0588869d92d85b914ec4a3bd3705a4161e848d92a6cde61c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-17T10:56:25.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 236912 9.59 MB/sec execute 497 sec latency 54.337 ms 2024-09-17T10:56:25.918 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:25 smithi149 podman[55270]: 2024-09-17 10:56:25.684756773 +0000 UTC m=+0.716052320 container remove 9161ab8a8bc64b0588869d92d85b914ec4a3bd3705a4161e848d92a6cde61c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-7498493, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-17T10:56:25.918 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:25 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-17T10:56:25.918 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:25 smithi149 systemd[1]: Stopped Ceph osd.5 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:56:25.918 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:25 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 45.628s CPU time. 2024-09-17T10:56:25.918 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:25 smithi149 systemd[1]: Starting Ceph osd.5 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:56:26.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:26.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:26 smithi149 ceph-mon[49697]: pgmap v223: 97 pgs: 1 active+undersized, 10 peering, 7 stale+active+clean, 4 active+undersized+degraded, 75 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.3 MiB/s wr, 418 op/s; 25/1641 objects degraded (1.523%) 2024-09-17T10:56:26.207 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 podman[55368]: 2024-09-17 10:56:26.052818584 +0000 UTC m=+0.019363623 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:56:26.207 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 podman[55368]: 2024-09-17 10:56:26.152856756 +0000 UTC m=+0.119401785 container create e0018d49879668245a04e5ef158721681171ac195b9cf2f137ba0150a3584098 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T10:56:26.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:26.255 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:26 smithi006 ceph-mon[63965]: pgmap v223: 97 pgs: 1 active+undersized, 10 peering, 7 stale+active+clean, 4 active+undersized+degraded, 75 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.3 MiB/s wr, 418 op/s; 25/1641 objects degraded (1.523%) 2024-09-17T10:56:26.515 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 podman[55368]: 2024-09-17 10:56:26.422613089 +0000 UTC m=+0.389158126 container init e0018d49879668245a04e5ef158721681171ac195b9cf2f137ba0150a3584098 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-17T10:56:26.515 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 podman[55368]: 2024-09-17 10:56:26.426067089 +0000 UTC m=+0.392612119 container start e0018d49879668245a04e5ef158721681171ac195b9cf2f137ba0150a3584098 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T10:56:26.515 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 podman[55368]: 2024-09-17 10:56:26.514358363 +0000 UTC m=+0.480903404 container attach e0018d49879668245a04e5ef158721681171ac195b9cf2f137ba0150a3584098 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-7498493, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T10:56:26.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 237139 9.58 MB/sec execute 498 sec latency 50.157 ms 2024-09-17T10:56:26.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:56:26.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 bash[55368]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:56:26.957 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:56:26.957 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:26 smithi149 bash[55368]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:56:27.260 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:27 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 25/1641 objects degraded (1.523%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:27.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:27 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 25/1641 objects degraded (1.523%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:27.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 237415 9.58 MB/sec execute 499 sec latency 52.967 ms 2024-09-17T10:56:27.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T10:56:27.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 bash[55368]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T10:56:27.957 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T10:56:27.957 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 bash[55368]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T10:56:28.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:28 smithi006 ceph-mon[63965]: pgmap v224: 97 pgs: 4 active+undersized, 10 peering, 16 active+undersized+degraded, 67 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.4 MiB/s wr, 492 op/s; 130/1623 objects degraded (8.010%) 2024-09-17T10:56:28.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:28 smithi149 ceph-mon[49697]: pgmap v224: 97 pgs: 4 active+undersized, 10 peering, 16 active+undersized+degraded, 67 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.4 MiB/s wr, 492 op/s; 130/1623 objects degraded (8.010%) 2024-09-17T10:56:28.456 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 bash[55368]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 bash[55368]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 bash[55368]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 bash[55368]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate[55383]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:27 smithi149 bash[55368]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T10:56:28.457 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:28 smithi149 podman[55368]: 2024-09-17 10:56:28.000524257 +0000 UTC m=+1.967069299 container died e0018d49879668245a04e5ef158721681171ac195b9cf2f137ba0150a3584098 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, RELEASE=main-7498493, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:56:28.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 237766 9.58 MB/sec execute 500 sec latency 44.794 ms 2024-09-17T10:56:29.321 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:29 smithi149 podman[55515]: 2024-09-17 10:56:29.069591082 +0000 UTC m=+1.057604895 container remove e0018d49879668245a04e5ef158721681171ac195b9cf2f137ba0150a3584098 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2) 2024-09-17T10:56:29.321 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:29 smithi149 podman[55570]: 2024-09-17 10:56:29.205870633 +0000 UTC m=+0.022627886 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:56:29.321 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:29 smithi149 podman[55570]: 2024-09-17 10:56:29.320820818 +0000 UTC m=+0.137578061 container create 144416f89b4a49ae129dafc8245910e18b49da7c741c87653ed754be4bc6ae9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-17T10:56:29.636 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:29 smithi149 podman[55570]: 2024-09-17 10:56:29.548222798 +0000 UTC m=+0.364980052 container init 144416f89b4a49ae129dafc8245910e18b49da7c741c87653ed754be4bc6ae9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T10:56:29.636 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:29 smithi149 podman[55570]: 2024-09-17 10:56:29.55182476 +0000 UTC m=+0.368582003 container start 144416f89b4a49ae129dafc8245910e18b49da7c741c87653ed754be4bc6ae9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-17T10:56:29.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 238078 9.59 MB/sec execute 501 sec latency 59.890 ms 2024-09-17T10:56:29.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:29 smithi149 ceph-osd[55589]: -- 172.21.15.149:0/3122650098 <== mon.1 v2:172.21.15.149:3300/0 3 ==== mon_map magic: 0 ==== 357+0+0 (secure 0 0 0) 0x5641b61c61a0 con 0x5641b53a9c00 2024-09-17T10:56:29.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:29 smithi149 bash[55570]: 144416f89b4a49ae129dafc8245910e18b49da7c741c87653ed754be4bc6ae9d 2024-09-17T10:56:29.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:29 smithi149 systemd[1]: Started Ceph osd.5 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:56:30.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:30 smithi006 ceph-mon[63965]: pgmap v225: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.9 MiB/s wr, 444 op/s; 174/1623 objects degraded (10.721%) 2024-09-17T10:56:30.826 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:30.826 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:30.826 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:56:30.826 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:30.826 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:56:30.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 238351 9.59 MB/sec execute 502 sec latency 58.253 ms 2024-09-17T10:56:31.206 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:30 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:56:30.724+0000 7f48bd49f640 -1 Falling back to public interface 2024-09-17T10:56:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:30 smithi149 ceph-mon[49697]: pgmap v225: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.9 MiB/s wr, 444 op/s; 174/1623 objects degraded (10.721%) 2024-09-17T10:56:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:31.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:56:31.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:31.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:56:31.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 238617 9.58 MB/sec execute 503 sec latency 51.264 ms 2024-09-17T10:56:31.844 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:31 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 170/1656 objects degraded (10.266%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:32.126 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:31 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 170/1656 objects degraded (10.266%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:32.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 238950 9.56 MB/sec execute 504 sec latency 66.674 ms 2024-09-17T10:56:32.876 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:32 smithi006 ceph-mon[63965]: pgmap v226: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.1 MiB/s wr, 338 op/s; 170/1656 objects degraded (10.266%) 2024-09-17T10:56:33.167 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:32 smithi149 ceph-mon[49697]: pgmap v226: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.1 MiB/s wr, 338 op/s; 170/1656 objects degraded (10.266%) 2024-09-17T10:56:33.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 239171 9.55 MB/sec execute 505 sec latency 50.911 ms 2024-09-17T10:56:34.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:34.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:34.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:34 smithi006 ceph-mon[63965]: pgmap v227: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 448 op/s; 173/1671 objects degraded (10.353%) 2024-09-17T10:56:34.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:34.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:34 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:34.667 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:34.668 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:34.668 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:34 smithi149 ceph-mon[49697]: pgmap v227: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 448 op/s; 173/1671 objects degraded (10.353%) 2024-09-17T10:56:34.668 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:34.668 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:34 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:34.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 239406 9.55 MB/sec execute 506 sec latency 48.137 ms 2024-09-17T10:56:35.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 239715 9.56 MB/sec execute 507 sec latency 63.033 ms 2024-09-17T10:56:36.722 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:36 smithi006 ceph-mon[63965]: pgmap v228: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 381 op/s; 172/1653 objects degraded (10.405%) 2024-09-17T10:56:36.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 240095 9.56 MB/sec execute 508 sec latency 61.303 ms 2024-09-17T10:56:36.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:36 smithi149 ceph-mon[49697]: pgmap v228: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 381 op/s; 172/1653 objects degraded (10.405%) 2024-09-17T10:56:37.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:37 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 172/1653 objects degraded (10.405%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:37.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:37.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:37.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:56:37.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:56:37.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:37.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:37 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:37.758 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:37 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 172/1653 objects degraded (10.405%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:37.758 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:37.758 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:37.758 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:56:37.758 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:56:37.758 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:37.758 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:37 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:37.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 240458 9.55 MB/sec execute 509 sec latency 54.982 ms 2024-09-17T10:56:38.802 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:38 smithi006 ceph-mon[63965]: pgmap v229: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 499 op/s; 168/1581 objects degraded (10.626%) 2024-09-17T10:56:38.802 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:56:38.802 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:38.802 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:38.802 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:38.802 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:38 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T10:56:38.835 INFO:teuthology.orchestra.run.smithi006.stdout: 5 240657 9.54 MB/sec execute 510 sec latency 75.250 ms 2024-09-17T10:56:38.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:38 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:56:38.744+0000 7f48bd49f640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-17T10:56:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:38 smithi149 ceph-mon[49697]: pgmap v229: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 499 op/s; 168/1581 objects degraded (10.626%) 2024-09-17T10:56:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:56:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:38.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:38.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:38.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:38 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T10:56:39.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 240821 9.53 MB/sec execute 511 sec latency 62.003 ms 2024-09-17T10:56:39.861 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:39 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T10:56:39.861 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:39 smithi006 ceph-mon[63965]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T10:56:39.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:39 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T10:56:39.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:39 smithi149 ceph-mon[49697]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T10:56:40.456 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:39 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:56:39.955+0000 7f48bd49f640 -1 osd.5 127 log_to_monitors true 2024-09-17T10:56:40.643 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:40 smithi006 ceph-mon[63965]: pgmap v230: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 362 op/s; 171/1587 objects degraded (10.775%) 2024-09-17T10:56:40.644 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:40 smithi006 ceph-mon[63965]: from='osd.5 [v2:172.21.15.149:6808/1661965228,v1:172.21.15.149:6809/1661965228]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T10:56:40.644 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:40 smithi006 ceph-mon[63965]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T10:56:40.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 241050 9.53 MB/sec execute 512 sec latency 70.182 ms 2024-09-17T10:56:40.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:56:40 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:56:40.705+0000 7f48b5249640 -1 osd.5 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:56:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:40 smithi149 ceph-mon[49697]: pgmap v230: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 362 op/s; 171/1587 objects degraded (10.775%) 2024-09-17T10:56:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:40 smithi149 ceph-mon[49697]: from='osd.5 [v2:172.21.15.149:6808/1661965228,v1:172.21.15.149:6809/1661965228]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T10:56:40.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:40 smithi149 ceph-mon[49697]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T10:56:41.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 241366 9.53 MB/sec execute 513 sec latency 136.565 ms 2024-09-17T10:56:41.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:41 smithi006 ceph-mon[63965]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T10:56:41.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:41 smithi006 ceph-mon[63965]: osdmap e130: 8 total, 7 up, 8 in 2024-09-17T10:56:41.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:41 smithi006 ceph-mon[63965]: from='osd.5 [v2:172.21.15.149:6808/1661965228,v1:172.21.15.149:6809/1661965228]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:56:41.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:41 smithi006 ceph-mon[63965]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:56:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:41 smithi149 ceph-mon[49697]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T10:56:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:41 smithi149 ceph-mon[49697]: osdmap e130: 8 total, 7 up, 8 in 2024-09-17T10:56:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:41 smithi149 ceph-mon[49697]: from='osd.5 [v2:172.21.15.149:6808/1661965228,v1:172.21.15.149:6809/1661965228]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:56:41.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:41 smithi149 ceph-mon[49697]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:56:42.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:42 smithi006 ceph-mon[63965]: pgmap v232: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.4 MiB/s wr, 508 op/s; 179/1644 objects degraded (10.888%) 2024-09-17T10:56:42.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:42 smithi006 ceph-mon[63965]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:56:42.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:42 smithi006 ceph-mon[63965]: osd.5 [v2:172.21.15.149:6808/1661965228,v1:172.21.15.149:6809/1661965228] boot 2024-09-17T10:56:42.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:42 smithi006 ceph-mon[63965]: osdmap e131: 8 total, 8 up, 8 in 2024-09-17T10:56:42.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T10:56:42.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:42 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 179/1644 objects degraded (10.888%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:42.746 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:42 smithi006 ceph-mon[63965]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T10:56:42.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 241559 9.52 MB/sec execute 514 sec latency 642.402 ms 2024-09-17T10:56:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:42 smithi149 ceph-mon[49697]: pgmap v232: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.4 MiB/s wr, 508 op/s; 179/1644 objects degraded (10.888%) 2024-09-17T10:56:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:42 smithi149 ceph-mon[49697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:56:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:42 smithi149 ceph-mon[49697]: osd.5 [v2:172.21.15.149:6808/1661965228,v1:172.21.15.149:6809/1661965228] boot 2024-09-17T10:56:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:42 smithi149 ceph-mon[49697]: osdmap e131: 8 total, 8 up, 8 in 2024-09-17T10:56:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T10:56:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:42 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 179/1644 objects degraded (10.888%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:42.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:42 smithi149 ceph-mon[49697]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T10:56:43.324 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:56:43.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 241755 9.52 MB/sec execute 515 sec latency 1121.256 ms 2024-09-17T10:56:44.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:44 smithi006 ceph-mon[63965]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T10:56:44.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:44 smithi006 ceph-mon[63965]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:44.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:44 smithi006 ceph-mon[63965]: pgmap v236: 97 pgs: 1 remapped+peering, 3 peering, 3 active+undersized, 17 active+undersized+degraded, 73 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 561 op/s; 135/1659 objects degraded (8.137%); 229 KiB/s, 9 objects/s recovering 2024-09-17T10:56:44.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:44 smithi006 ceph-mon[63965]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:44.383 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:56:44.383 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (8m) 88s ago 17m 18.0M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:56:44.383 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (4m) 88s ago 17m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:56:44.383 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (4m) 10s ago 16m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (7m) 88s ago 16m 42.7M - dad864ee21e9 89df198e1a9b 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (11m) 88s ago 11m 38.0M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (11m) 10s ago 11m 15.8M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (7m) 88s ago 18m 518M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (6m) 10s ago 16m 457M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (5m) 88s ago 18m 52.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (5m) 10s ago 16m 43.4M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (10m) 88s ago 10m 286M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (10m) 10s ago 10m 88.5M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (7m) 88s ago 16m 9382k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (7m) 10s ago 16m 11.7M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (3m) 88s ago 15m 334M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:56:44.384 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (3m) 88s ago 15m 254M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:56:44.385 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (2m) 88s ago 14m 213M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:56:44.385 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (96s) 88s ago 14m 11.7M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T10:56:44.385 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (52s) 10s ago 13m 261M 4096M 19.3.0-4937-g74984932 6c1fcab85602 016956823ff1 2024-09-17T10:56:44.385 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (15s) 10s ago 13m 14.3M 4096M 19.3.0-4937-g74984932 6c1fcab85602 144416f89b4a 2024-09-17T10:56:44.385 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (13m) 10s ago 13m 488M 4096M 17.2.0 e1d6a67b021e e61ff1670f97 2024-09-17T10:56:44.385 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (12m) 10s ago 12m 713M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:56:44.385 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (6m) 88s ago 16m 53.3M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:56:44.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:44 smithi149 ceph-mon[49697]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T10:56:44.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:44 smithi149 ceph-mon[49697]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:44.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:44 smithi149 ceph-mon[49697]: pgmap v236: 97 pgs: 1 remapped+peering, 3 peering, 3 active+undersized, 17 active+undersized+degraded, 73 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 561 op/s; 135/1659 objects degraded (8.137%); 229 KiB/s, 9 objects/s recovering 2024-09-17T10:56:44.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:44 smithi149 ceph-mon[49697]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:44.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 242034 9.51 MB/sec execute 516 sec latency 51.089 ms 2024-09-17T10:56:45.078 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:56:45.078 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 6 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:56:45.079 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T10:56:45.080 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 10 2024-09-17T10:56:45.080 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:56:45.080 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:56:45.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:45 smithi006 ceph-mon[63965]: osdmap e134: 8 total, 8 up, 8 in 2024-09-17T10:56:45.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:45 smithi006 ceph-mon[63965]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:45.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:45.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:56:45.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:45 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1374268702' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:45.623 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "12/23 daemons upgraded", 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:56:45.624 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:56:45.625 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:56:45.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:45 smithi149 ceph-mon[49697]: osdmap e134: 8 total, 8 up, 8 in 2024-09-17T10:56:45.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:45 smithi149 ceph-mon[49697]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:45.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:45.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:56:45.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:45 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/1374268702' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:56:45.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 242305 9.51 MB/sec execute 517 sec latency 52.269 ms 2024-09-17T10:56:46.301 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN Degraded data redundancy: 113/1650 objects degraded (6.848%), 14 pgs degraded 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 113/1650 objects degraded (6.848%), 14 pgs degraded 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-17T10:56:46.302 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-17T10:56:46.303 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-17T10:56:46.355 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:46 smithi006 ceph-mon[63965]: osdmap e135: 8 total, 8 up, 8 in 2024-09-17T10:56:46.356 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:46 smithi006 ceph-mon[63965]: pgmap v239: 97 pgs: 1 remapped+peering, 3 peering, 2 active+undersized, 14 active+undersized+degraded, 77 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.3 MiB/s wr, 231 op/s; 113/1650 objects degraded (6.848%); 245 KiB/s, 15 objects/s recovering 2024-09-17T10:56:46.356 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:46 smithi006 ceph-mon[63965]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:46 smithi149 ceph-mon[49697]: osdmap e135: 8 total, 8 up, 8 in 2024-09-17T10:56:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:46 smithi149 ceph-mon[49697]: pgmap v239: 97 pgs: 1 remapped+peering, 3 peering, 2 active+undersized, 14 active+undersized+degraded, 77 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.3 MiB/s wr, 231 op/s; 113/1650 objects degraded (6.848%); 245 KiB/s, 15 objects/s recovering 2024-09-17T10:56:46.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:46 smithi149 ceph-mon[49697]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:56:46.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 242576 9.49 MB/sec execute 518 sec latency 61.599 ms 2024-09-17T10:56:47.375 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:47 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1853915328' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:56:47.376 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:47 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 113/1650 objects degraded (6.848%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:47.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:47 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/1853915328' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:56:47.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:47 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 113/1650 objects degraded (6.848%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T10:56:47.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 242872 9.49 MB/sec execute 519 sec latency 42.806 ms 2024-09-17T10:56:48.409 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:48 smithi006 ceph-mon[63965]: pgmap v240: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.8 MiB/s wr, 231 op/s; 1.6 MiB/s, 39 objects/s recovering 2024-09-17T10:56:48.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:48 smithi149 ceph-mon[49697]: pgmap v240: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.8 MiB/s wr, 231 op/s; 1.6 MiB/s, 39 objects/s recovering 2024-09-17T10:56:48.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 243192 9.49 MB/sec execute 520 sec latency 58.489 ms 2024-09-17T10:56:49.444 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:49 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 113/1650 objects degraded (6.848%), 14 pgs degraded) 2024-09-17T10:56:49.445 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:49 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:56:49.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:49 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 113/1650 objects degraded (6.848%), 14 pgs degraded) 2024-09-17T10:56:49.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:49 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:56:49.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 243501 9.49 MB/sec execute 521 sec latency 47.667 ms 2024-09-17T10:56:50.462 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:50 smithi006 ceph-mon[63965]: pgmap v241: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.9 MiB/s wr, 330 op/s; 1.2 MiB/s, 26 objects/s recovering 2024-09-17T10:56:50.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:50 smithi149 ceph-mon[49697]: pgmap v241: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.9 MiB/s wr, 330 op/s; 1.2 MiB/s, 26 objects/s recovering 2024-09-17T10:56:50.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 243750 9.48 MB/sec execute 522 sec latency 74.833 ms 2024-09-17T10:56:51.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 244026 9.48 MB/sec execute 523 sec latency 51.418 ms 2024-09-17T10:56:52.752 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:52 smithi006 ceph-mon[63965]: pgmap v242: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.8 MiB/s wr, 377 op/s; 962 KiB/s, 21 objects/s recovering 2024-09-17T10:56:52.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 244275 9.48 MB/sec execute 524 sec latency 65.746 ms 2024-09-17T10:56:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:52 smithi149 ceph-mon[49697]: pgmap v242: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.8 MiB/s wr, 377 op/s; 962 KiB/s, 21 objects/s recovering 2024-09-17T10:56:53.689 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:53 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T10:56:53.805 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:53 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T10:56:53.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 244569 9.47 MB/sec execute 525 sec latency 60.905 ms 2024-09-17T10:56:54.819 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:54 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T10:56:54.819 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:54 smithi006 ceph-mon[63965]: Upgrade: osd.6 is safe to restart 2024-09-17T10:56:54.820 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:54 smithi006 ceph-mon[63965]: pgmap v243: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 374 op/s; 832 KiB/s, 18 objects/s recovering 2024-09-17T10:56:54.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 244827 9.47 MB/sec execute 526 sec latency 63.449 ms 2024-09-17T10:56:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:54 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T10:56:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:54 smithi149 ceph-mon[49697]: Upgrade: osd.6 is safe to restart 2024-09-17T10:56:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:54 smithi149 ceph-mon[49697]: pgmap v243: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 374 op/s; 832 KiB/s, 18 objects/s recovering 2024-09-17T10:56:55.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 245094 9.47 MB/sec execute 527 sec latency 72.556 ms 2024-09-17T10:56:55.841 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:55 smithi006 ceph-mon[63965]: Upgrade: Updating osd.6 2024-09-17T10:56:55.841 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:55.841 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T10:56:55.841 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:56:55.842 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:55 smithi006 ceph-mon[63965]: Deploying daemon osd.6 on smithi149 2024-09-17T10:56:55.842 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:55.936 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:55 smithi149 ceph-mon[49697]: Upgrade: Updating osd.6 2024-09-17T10:56:55.936 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:55.936 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T10:56:55.936 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:56:55.936 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:55 smithi149 ceph-mon[49697]: Deploying daemon osd.6 on smithi149 2024-09-17T10:56:55.937 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:56:56.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 245373 9.47 MB/sec execute 528 sec latency 71.705 ms 2024-09-17T10:56:56.867 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:56 smithi149 ceph-mon[49697]: pgmap v244: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 409 op/s; 751 KiB/s, 16 objects/s recovering 2024-09-17T10:56:56.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:56 smithi006 ceph-mon[63965]: pgmap v244: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 409 op/s; 751 KiB/s, 16 objects/s recovering 2024-09-17T10:56:57.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 245549 9.45 MB/sec execute 529 sec latency 189.408 ms 2024-09-17T10:56:58.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:58 smithi006 ceph-mon[63965]: pgmap v245: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.4 MiB/s wr, 436 op/s; 639 KiB/s, 12 objects/s recovering 2024-09-17T10:56:58.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:58 smithi149 ceph-mon[49697]: pgmap v245: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.4 MiB/s wr, 436 op/s; 639 KiB/s, 12 objects/s recovering 2024-09-17T10:56:58.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 245838 9.45 MB/sec execute 530 sec latency 64.724 ms 2024-09-17T10:56:59.456 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:56:59 smithi149 systemd[1]: Stopping Ceph osd.6 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:56:59.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 246004 9.44 MB/sec execute 531 sec latency 328.011 ms 2024-09-17T10:56:59.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:56:59 smithi006 ceph-mon[63965]: osd.6 marked itself down and dead 2024-09-17T10:56:59.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:56:59 smithi149 ceph-mon[49697]: osd.6 marked itself down and dead 2024-09-17T10:56:59.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:56:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[36188]: 2024-09-17T10:56:59.462+0000 7f36abfaf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:56:59.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:56:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[36188]: 2024-09-17T10:56:59.462+0000 7f36abfaf700 -1 osd.6 135 *** Got signal Terminated *** 2024-09-17T10:56:59.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:56:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[36188]: 2024-09-17T10:56:59.462+0000 7f36abfaf700 -1 osd.6 135 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T10:57:00.455 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:00 smithi149 podman[57138]: 2024-09-17 10:57:00.156923546 +0000 UTC m=+0.806896498 container died e61ff1670f9780a8d856ebee44df92db160696a7b492dad1bfbd60687f7f8336 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, RELEASE=HEAD, architecture=x86_64, name=centos-stream, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream) 2024-09-17T10:57:00.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:00 smithi149 ceph-mon[49697]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:57:00.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:00 smithi149 ceph-mon[49697]: osdmap e136: 8 total, 7 up, 8 in 2024-09-17T10:57:00.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:00 smithi149 ceph-mon[49697]: pgmap v247: 97 pgs: 10 stale+active+clean, 87 active+clean; 351 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 343 op/s 2024-09-17T10:57:00.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:00.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:57:00.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:00 smithi149 ceph-mon[49697]: osdmap e137: 8 total, 7 up, 8 in 2024-09-17T10:57:00.707 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:00 smithi149 podman[57138]: 2024-09-17 10:57:00.545159938 +0000 UTC m=+1.195132879 container cleanup e61ff1670f9780a8d856ebee44df92db160696a7b492dad1bfbd60687f7f8336 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, vcs-type=git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, name=centos-stream, ceph=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, release=754, distribution-scope=public) 2024-09-17T10:57:00.707 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:00 smithi149 bash[57138]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6 2024-09-17T10:57:00.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:00 smithi006 ceph-mon[63965]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:57:00.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:00 smithi006 ceph-mon[63965]: osdmap e136: 8 total, 7 up, 8 in 2024-09-17T10:57:00.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:00 smithi006 ceph-mon[63965]: pgmap v247: 97 pgs: 10 stale+active+clean, 87 active+clean; 351 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 343 op/s 2024-09-17T10:57:00.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:00.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:57:00.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:00 smithi006 ceph-mon[63965]: osdmap e137: 8 total, 7 up, 8 in 2024-09-17T10:57:00.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 246082 9.42 MB/sec execute 532 sec latency 1096.865 ms 2024-09-17T10:57:01.206 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:00 smithi149 podman[57154]: 2024-09-17 10:57:00.838691034 +0000 UTC m=+0.679526253 container remove e61ff1670f9780a8d856ebee44df92db160696a7b492dad1bfbd60687f7f8336 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.openshift.expose-services=, release=754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.component=centos-stream-container, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, RELEASE=HEAD) 2024-09-17T10:57:01.470 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:01 smithi149 podman[57241]: 2024-09-17 10:57:01.121036338 +0000 UTC m=+0.018496756 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:57:01.470 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:01 smithi149 podman[57241]: 2024-09-17 10:57:01.231668168 +0000 UTC m=+0.129128580 container create d6ed6e9b57a807ccfcc37c20e304c36806951a619e320f82ff07a44c9bbf613f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-17T10:57:01.471 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:01 smithi149 podman[57241]: 2024-09-17 10:57:01.469819775 +0000 UTC m=+0.367280191 container init d6ed6e9b57a807ccfcc37c20e304c36806951a619e320f82ff07a44c9bbf613f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-7498493, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-17T10:57:01.796 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:01 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 107/1572 objects degraded (6.807%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:01.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 246384 9.42 MB/sec execute 533 sec latency 38.075 ms 2024-09-17T10:57:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:01 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 107/1572 objects degraded (6.807%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:01.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:01 smithi149 podman[57241]: 2024-09-17 10:57:01.474360469 +0000 UTC m=+0.371820882 container start d6ed6e9b57a807ccfcc37c20e304c36806951a619e320f82ff07a44c9bbf613f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:57:01.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:01 smithi149 podman[57241]: 2024-09-17 10:57:01.578318048 +0000 UTC m=+0.475778482 container attach d6ed6e9b57a807ccfcc37c20e304c36806951a619e320f82ff07a44c9bbf613f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T10:57:01.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:01 smithi149 podman[57241]: 2024-09-17 10:57:01.714720032 +0000 UTC m=+0.612180446 container died d6ed6e9b57a807ccfcc37c20e304c36806951a619e320f82ff07a44c9bbf613f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, RELEASE=main-7498493, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T10:57:02.608 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:02 smithi149 podman[57264]: 2024-09-17 10:57:02.383059028 +0000 UTC m=+0.660418934 container remove d6ed6e9b57a807ccfcc37c20e304c36806951a619e320f82ff07a44c9bbf613f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-7498493, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-09-17T10:57:02.608 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:02 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-17T10:57:02.608 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:02 smithi149 systemd[1]: Stopped Ceph osd.6 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:57:02.608 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:02 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.6.service: Consumed 4min 23.458s CPU time. 2024-09-17T10:57:02.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 246693 9.42 MB/sec execute 534 sec latency 63.361 ms 2024-09-17T10:57:02.862 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:02 smithi006 ceph-mon[63965]: pgmap v249: 97 pgs: 4 active+undersized, 6 stale+active+clean, 15 active+undersized+degraded, 72 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.7 MiB/s wr, 409 op/s; 107/1572 objects degraded (6.807%) 2024-09-17T10:57:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:02 smithi149 ceph-mon[49697]: pgmap v249: 97 pgs: 4 active+undersized, 6 stale+active+clean, 15 active+undersized+degraded, 72 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.7 MiB/s wr, 409 op/s; 107/1572 objects degraded (6.807%) 2024-09-17T10:57:02.957 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:02 smithi149 systemd[1]: Starting Ceph osd.6 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:57:02.957 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:02 smithi149 podman[57362]: 2024-09-17 10:57:02.732104625 +0000 UTC m=+0.016786773 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:57:02.957 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:02 smithi149 podman[57362]: 2024-09-17 10:57:02.843026997 +0000 UTC m=+0.127709185 container create c9a75c9b3e8c97a42b284458badfad129436cc07ef39a7531c700191ffe93c96 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True) 2024-09-17T10:57:03.230 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:03 smithi149 podman[57362]: 2024-09-17 10:57:03.129136922 +0000 UTC m=+0.413819071 container init c9a75c9b3e8c97a42b284458badfad129436cc07ef39a7531c700191ffe93c96 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-7498493) 2024-09-17T10:57:03.230 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:03 smithi149 podman[57362]: 2024-09-17 10:57:03.132554933 +0000 UTC m=+0.417237084 container start c9a75c9b3e8c97a42b284458badfad129436cc07ef39a7531c700191ffe93c96 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-17T10:57:03.230 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:03 smithi149 podman[57362]: 2024-09-17 10:57:03.22907775 +0000 UTC m=+0.513759911 container attach c9a75c9b3e8c97a42b284458badfad129436cc07ef39a7531c700191ffe93c96 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate, RELEASE=main-7498493, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T10:57:03.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:57:03.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:03 smithi149 bash[57362]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:57:03.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:57:03.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:03 smithi149 bash[57362]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:57:03.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 247013 9.42 MB/sec execute 535 sec latency 46.396 ms 2024-09-17T10:57:04.368 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T10:57:04.368 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 bash[57362]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T10:57:04.368 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T10:57:04.368 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 bash[57362]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T10:57:04.659 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 bash[57362]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 bash[57362]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 bash[57362]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 bash[57362]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate[57378]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 bash[57362]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T10:57:04.660 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:04 smithi149 podman[57362]: 2024-09-17 10:57:04.427801133 +0000 UTC m=+1.712483279 container died c9a75c9b3e8c97a42b284458badfad129436cc07ef39a7531c700191ffe93c96 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-17T10:57:04.690 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:04 smithi006 ceph-mon[63965]: pgmap v250: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.0 MiB/s wr, 396 op/s; 189/1587 objects degraded (11.909%) 2024-09-17T10:57:04.836 INFO:teuthology.orchestra.run.smithi006.stdout: 5 247273 9.41 MB/sec execute 536 sec latency 64.529 ms 2024-09-17T10:57:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:04 smithi149 ceph-mon[49697]: pgmap v250: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.0 MiB/s wr, 396 op/s; 189/1587 objects degraded (11.909%) 2024-09-17T10:57:05.436 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:05 smithi149 podman[57514]: 2024-09-17 10:57:05.084097392 +0000 UTC m=+0.649770705 container remove c9a75c9b3e8c97a42b284458badfad129436cc07ef39a7531c700191ffe93c96 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-17T10:57:05.437 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:05 smithi149 podman[57573]: 2024-09-17 10:57:05.196960579 +0000 UTC m=+0.017434457 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:57:05.437 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:05 smithi149 podman[57573]: 2024-09-17 10:57:05.31012937 +0000 UTC m=+0.130603237 container create 2908fd9a4510345ec32913f8c41c9abd3b251f6eae1c715b525e882528b7c40a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-17T10:57:05.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:05 smithi149 podman[57573]: 2024-09-17 10:57:05.546367251 +0000 UTC m=+0.366841119 container init 2908fd9a4510345ec32913f8c41c9abd3b251f6eae1c715b525e882528b7c40a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-7498493, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:57:05.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:05 smithi149 podman[57573]: 2024-09-17 10:57:05.549786147 +0000 UTC m=+0.370260013 container start 2908fd9a4510345ec32913f8c41c9abd3b251f6eae1c715b525e882528b7c40a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-17T10:57:05.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:05 smithi149 ceph-osd[57592]: -- 172.21.15.149:0/3306482186 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bb25f8e1e0 con 0x55bb25f1e000 2024-09-17T10:57:05.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:05 smithi149 bash[57573]: 2908fd9a4510345ec32913f8c41c9abd3b251f6eae1c715b525e882528b7c40a 2024-09-17T10:57:05.707 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:05 smithi149 systemd[1]: Started Ceph osd.6 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:57:05.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 247543 9.41 MB/sec execute 537 sec latency 44.820 ms 2024-09-17T10:57:06.717 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:06 smithi006 ceph-mon[63965]: pgmap v251: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.5 MiB/s wr, 258 op/s; 189/1587 objects degraded (11.909%) 2024-09-17T10:57:06.717 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:06 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:06.717 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:06 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:06.717 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:06 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:57:06.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 247769 9.41 MB/sec execute 538 sec latency 82.410 ms 2024-09-17T10:57:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:06 smithi149 ceph-mon[49697]: pgmap v251: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.5 MiB/s wr, 258 op/s; 189/1587 objects degraded (11.909%) 2024-09-17T10:57:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:06 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:06 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:06.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:06 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:57:07.705 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:07 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T10:57:07.433+0000 7fc6ab877640 -1 Falling back to public interface 2024-09-17T10:57:07.731 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:07 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 189/1587 objects degraded (11.909%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:07.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248031 9.40 MB/sec execute 539 sec latency 57.496 ms 2024-09-17T10:57:07.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:07 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 189/1587 objects degraded (11.909%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:08.750 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:08 smithi006 ceph-mon[63965]: pgmap v252: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.3 MiB/s wr, 344 op/s; 196/1620 objects degraded (12.099%) 2024-09-17T10:57:08.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248259 9.39 MB/sec execute 540 sec latency 54.767 ms 2024-09-17T10:57:08.882 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:08 smithi149 ceph-mon[49697]: pgmap v252: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.3 MiB/s wr, 344 op/s; 196/1620 objects degraded (12.099%) 2024-09-17T10:57:09.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248483 9.39 MB/sec execute 541 sec latency 79.951 ms 2024-09-17T10:57:10.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:10.542 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:10.542 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:10 smithi006 ceph-mon[63965]: pgmap v253: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 428 op/s; 187/1626 objects degraded (11.501%) 2024-09-17T10:57:10.542 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:10.542 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:10.575 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:10 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:10.575 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:10 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:10.575 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:10 smithi149 ceph-mon[49697]: pgmap v253: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 428 op/s; 187/1626 objects degraded (11.501%) 2024-09-17T10:57:10.575 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:10 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:10.575 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:10 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:10.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248736 9.39 MB/sec execute 542 sec latency 47.981 ms 2024-09-17T10:57:11.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 249064 9.39 MB/sec execute 543 sec latency 59.769 ms 2024-09-17T10:57:12.836 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:12 smithi006 ceph-mon[63965]: pgmap v254: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 418 op/s; 185/1632 objects degraded (11.336%) 2024-09-17T10:57:12.837 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:12 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 185/1632 objects degraded (11.336%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:12.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 249455 9.39 MB/sec execute 544 sec latency 48.106 ms 2024-09-17T10:57:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:12 smithi149 ceph-mon[49697]: pgmap v254: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 418 op/s; 185/1632 objects degraded (11.336%) 2024-09-17T10:57:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:12 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 185/1632 objects degraded (11.336%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:13.837 INFO:teuthology.orchestra.run.smithi006.stdout: 5 249729 9.38 MB/sec execute 545 sec latency 53.469 ms 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.2 MiB/s wr, 420 op/s; 191/1632 objects degraded (11.703%) 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:57:14.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:14.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:14.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:14.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T10:57:14.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T10:57:14.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:14 smithi006 ceph-mon[63965]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T10:57:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:14.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.2 MiB/s wr, 420 op/s; 191/1632 objects degraded (11.703%) 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T10:57:14.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:14 smithi149 ceph-mon[49697]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T10:57:14.843 INFO:teuthology.orchestra.run.smithi006.stdout: 5 250025 9.38 MB/sec execute 546 sec latency 53.050 ms 2024-09-17T10:57:15.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:15 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T10:57:15.392+0000 7fc6ab877640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-17T10:57:15.843 INFO:teuthology.orchestra.run.smithi006.stdout: 5 250355 9.37 MB/sec execute 547 sec latency 45.573 ms 2024-09-17T10:57:16.193 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:16.194 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:57:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:57:16.843 INFO:teuthology.orchestra.run.smithi006.stdout: 5 250681 9.38 MB/sec execute 548 sec latency 45.184 ms 2024-09-17T10:57:16.900 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:57:16.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:16 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T10:57:16.588+0000 7fc6ab877640 -1 osd.6 135 log_to_monitors true 2024-09-17T10:57:17.219 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:17 smithi006 ceph-mon[63965]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.7 MiB/s wr, 368 op/s; 191/1632 objects degraded (11.703%) 2024-09-17T10:57:17.220 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:17 smithi006 ceph-mon[63965]: from='osd.6 [v2:172.21.15.149:6816/3140150439,v1:172.21.15.149:6817/3140150439]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T10:57:17.220 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:17 smithi006 ceph-mon[63965]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T10:57:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:17 smithi149 ceph-mon[49697]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.7 MiB/s wr, 368 op/s; 191/1632 objects degraded (11.703%) 2024-09-17T10:57:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:17 smithi149 ceph-mon[49697]: from='osd.6 [v2:172.21.15.149:6816/3140150439,v1:172.21.15.149:6817/3140150439]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T10:57:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:17 smithi149 ceph-mon[49697]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T10:57:17.843 INFO:teuthology.orchestra.run.smithi006.stdout: 5 250857 9.37 MB/sec execute 549 sec latency 155.253 ms 2024-09-17T10:57:17.854 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (8m) 2m ago 17m 18.0M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (5m) 2m ago 17m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (5m) 8s ago 17m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (8m) 2m ago 17m 42.7M - dad864ee21e9 89df198e1a9b 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (11m) 2m ago 11m 38.0M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (11m) 8s ago 11m 15.9M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (7m) 2m ago 18m 518M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (6m) 8s ago 17m 457M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (6m) 2m ago 19m 52.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (6m) 8s ago 16m 44.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:57:17.855 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (11m) 2m ago 11m 286M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (11m) 8s ago 11m 88.6M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (8m) 2m ago 17m 9382k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (7m) 8s ago 16m 12.2M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (4m) 2m ago 16m 334M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (3m) 2m ago 15m 254M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (2m) 2m ago 15m 213M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (2m) 2m ago 14m 11.7M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (85s) 8s ago 14m 291M 4096M 19.3.0-4937-g74984932 6c1fcab85602 016956823ff1 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (48s) 8s ago 14m 250M 4096M 19.3.0-4937-g74984932 6c1fcab85602 144416f89b4a 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (12s) 8s ago 13m 12.1M 4096M 19.3.0-4937-g74984932 6c1fcab85602 2908fd9a4510 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (13m) 8s ago 13m 762M 4096M 17.2.0 e1d6a67b021e 35f4f08ff1ae 2024-09-17T10:57:17.856 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (6m) 2m ago 16m 53.3M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:57:17.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 10:57:17 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T10:57:17.522+0000 7fc69f619640 -1 osd.6 135 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:57:18.241 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:18 smithi006 ceph-mon[63965]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:18.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:18 smithi006 ceph-mon[63965]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T10:57:18.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:18 smithi006 ceph-mon[63965]: from='osd.6 [v2:172.21.15.149:6816/3140150439,v1:172.21.15.149:6817/3140150439]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:57:18.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:18 smithi006 ceph-mon[63965]: osdmap e138: 8 total, 7 up, 8 in 2024-09-17T10:57:18.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:18 smithi006 ceph-mon[63965]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:57:18.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:18 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 191/1632 objects degraded (11.703%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:18.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:18 smithi006 ceph-mon[63965]: from='osd.6 ' entity='osd.6' 2024-09-17T10:57:18.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:18 smithi149 ceph-mon[49697]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:18.483 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:18 smithi149 ceph-mon[49697]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T10:57:18.483 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:18 smithi149 ceph-mon[49697]: from='osd.6 [v2:172.21.15.149:6816/3140150439,v1:172.21.15.149:6817/3140150439]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:57:18.483 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:18 smithi149 ceph-mon[49697]: osdmap e138: 8 total, 7 up, 8 in 2024-09-17T10:57:18.483 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:18 smithi149 ceph-mon[49697]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:57:18.483 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:18 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 191/1632 objects degraded (11.703%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:18.483 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:18 smithi149 ceph-mon[49697]: from='osd.6 ' entity='osd.6' 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:57:18.511 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 6 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 10 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:57:18.512 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:57:18.843 INFO:teuthology.orchestra.run.smithi006.stdout: 5 251140 9.36 MB/sec execute 550 sec latency 146.424 ms 2024-09-17T10:57:19.019 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:57:19.019 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "13/23 daemons upgraded", 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:57:19.020 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:57:19.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:19.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 491 op/s; 198/1686 objects degraded (11.744%) 2024-09-17T10:57:19.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:19.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/325044268' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:19.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:57:19.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: osd.6 [v2:172.21.15.149:6816/3140150439,v1:172.21.15.149:6817/3140150439] boot 2024-09-17T10:57:19.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: osdmap e139: 8 total, 8 up, 8 in 2024-09-17T10:57:19.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T10:57:19.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:19 smithi006 ceph-mon[63965]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 491 op/s; 198/1686 objects degraded (11.744%) 2024-09-17T10:57:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/325044268' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:57:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: osd.6 [v2:172.21.15.149:6816/3140150439,v1:172.21.15.149:6817/3140150439] boot 2024-09-17T10:57:19.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: osdmap e139: 8 total, 8 up, 8 in 2024-09-17T10:57:19.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T10:57:19.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:19 smithi149 ceph-mon[49697]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:19.547 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN Degraded data redundancy: 198/1686 objects degraded (11.744%), 25 pgs degraded 2024-09-17T10:57:19.548 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 198/1686 objects degraded (11.744%), 25 pgs degraded 2024-09-17T10:57:19.548 INFO:teuthology.orchestra.run.smithi006.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-17T10:57:19.548 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-17T10:57:19.548 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-17T10:57:19.548 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-17T10:57:19.549 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-17T10:57:19.549 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-17T10:57:19.549 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-17T10:57:19.549 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-17T10:57:19.549 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-17T10:57:19.549 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-17T10:57:19.550 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-17T10:57:19.550 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-17T10:57:19.550 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-17T10:57:19.550 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-17T10:57:19.550 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-17T10:57:19.550 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-17T10:57:19.551 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-17T10:57:19.551 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-17T10:57:19.551 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-17T10:57:19.551 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-17T10:57:19.551 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-17T10:57:19.551 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-17T10:57:19.551 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-17T10:57:19.552 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-17T10:57:19.552 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-17T10:57:19.843 INFO:teuthology.orchestra.run.smithi006.stdout: 5 251291 9.35 MB/sec execute 551 sec latency 1146.456 ms 2024-09-17T10:57:20.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:20 smithi006 ceph-mon[63965]: osdmap e140: 8 total, 8 up, 8 in 2024-09-17T10:57:20.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:20 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1395316972' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:57:20.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:20 smithi006 ceph-mon[63965]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.4 MiB/s wr, 329 op/s; 188/1632 objects degraded (11.520%) 2024-09-17T10:57:20.843 INFO:teuthology.orchestra.run.smithi006.stdout: 5 251503 9.35 MB/sec execute 552 sec latency 1875.211 ms 2024-09-17T10:57:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:20 smithi149 ceph-mon[49697]: osdmap e140: 8 total, 8 up, 8 in 2024-09-17T10:57:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:20 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/1395316972' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:57:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:20 smithi149 ceph-mon[49697]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.4 MiB/s wr, 329 op/s; 188/1632 objects degraded (11.520%) 2024-09-17T10:57:21.698 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:21 smithi006 ceph-mon[63965]: osdmap e141: 8 total, 8 up, 8 in 2024-09-17T10:57:21.843 INFO:teuthology.orchestra.run.smithi006.stdout: 5 251767 9.34 MB/sec execute 553 sec latency 207.755 ms 2024-09-17T10:57:21.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:21 smithi149 ceph-mon[49697]: osdmap e141: 8 total, 8 up, 8 in 2024-09-17T10:57:22.713 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:22 smithi006 ceph-mon[63965]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T10:57:22.714 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:22 smithi006 ceph-mon[63965]: pgmap v264: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+undersized, 12 active+undersized+degraded, 74 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 231 op/s; 108/1578 objects degraded (6.844%); 724 KiB/s, 19 objects/s recovering 2024-09-17T10:57:22.714 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:22 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 188/1632 objects degraded (11.520%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:22.714 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:22 smithi006 ceph-mon[63965]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T10:57:22.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 252074 9.34 MB/sec execute 554 sec latency 937.494 ms 2024-09-17T10:57:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:22 smithi149 ceph-mon[49697]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T10:57:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:22 smithi149 ceph-mon[49697]: pgmap v264: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+undersized, 12 active+undersized+degraded, 74 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 231 op/s; 108/1578 objects degraded (6.844%); 724 KiB/s, 19 objects/s recovering 2024-09-17T10:57:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:22 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 188/1632 objects degraded (11.520%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:22 smithi149 ceph-mon[49697]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T10:57:23.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 252383 9.34 MB/sec execute 555 sec latency 60.216 ms 2024-09-17T10:57:24.757 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:24 smithi006 ceph-mon[63965]: pgmap v266: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 350 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 495 op/s; 30/1632 objects degraded (1.838%); 1022 KiB/s, 60 keys/s, 44 objects/s recovering 2024-09-17T10:57:24.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 252661 9.34 MB/sec execute 556 sec latency 48.883 ms 2024-09-17T10:57:24.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:24 smithi149 ceph-mon[49697]: pgmap v266: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 350 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 495 op/s; 30/1632 objects degraded (1.838%); 1022 KiB/s, 60 keys/s, 44 objects/s recovering 2024-09-17T10:57:25.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 252944 9.35 MB/sec execute 557 sec latency 98.921 ms 2024-09-17T10:57:26.807 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:26 smithi006 ceph-mon[63965]: pgmap v267: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 350 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.0 MiB/s wr, 332 op/s; 30/1632 objects degraded (1.838%); 687 KiB/s, 40 keys/s, 29 objects/s recovering 2024-09-17T10:57:26.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 253198 9.34 MB/sec execute 558 sec latency 66.334 ms 2024-09-17T10:57:26.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:26 smithi149 ceph-mon[49697]: pgmap v267: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 350 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.0 MiB/s wr, 332 op/s; 30/1632 objects degraded (1.838%); 687 KiB/s, 40 keys/s, 29 objects/s recovering 2024-09-17T10:57:27.811 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:27 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 30/1632 objects degraded (1.838%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:27.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 253519 9.33 MB/sec execute 559 sec latency 43.446 ms 2024-09-17T10:57:27.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:27 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 30/1632 objects degraded (1.838%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:28.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:28 smithi006 ceph-mon[63965]: pgmap v268: 97 pgs: 97 active+clean; 351 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.7 MiB/s wr, 503 op/s; 1.3 MiB/s, 34 keys/s, 29 objects/s recovering 2024-09-17T10:57:28.826 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:28 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1632 objects degraded (1.838%), 3 pgs degraded) 2024-09-17T10:57:28.826 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:28 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:57:28.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 253820 9.33 MB/sec execute 560 sec latency 41.804 ms 2024-09-17T10:57:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:28 smithi149 ceph-mon[49697]: pgmap v268: 97 pgs: 97 active+clean; 351 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.7 MiB/s wr, 503 op/s; 1.3 MiB/s, 34 keys/s, 29 objects/s recovering 2024-09-17T10:57:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:28 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1632 objects degraded (1.838%), 3 pgs degraded) 2024-09-17T10:57:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:28 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:57:29.683 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T10:57:29.684 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:29 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T10:57:29.684 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:29 smithi149 ceph-mon[49697]: Upgrade: osd.7 is safe to restart 2024-09-17T10:57:29.841 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T10:57:29.842 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:29 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T10:57:29.842 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:29 smithi006 ceph-mon[63965]: Upgrade: osd.7 is safe to restart 2024-09-17T10:57:29.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 254143 9.32 MB/sec execute 561 sec latency 51.057 ms 2024-09-17T10:57:30.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 254472 9.32 MB/sec execute 562 sec latency 68.813 ms 2024-09-17T10:57:31.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:31 smithi006 ceph-mon[63965]: pgmap v269: 97 pgs: 97 active+clean; 353 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 465 op/s; 762 KiB/s, 30 keys/s, 16 objects/s recovering 2024-09-17T10:57:31.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:31 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:31.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:31 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:57:31.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:31 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:31.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:31 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T10:57:31.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:31 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:57:31.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:31 smithi149 ceph-mon[49697]: pgmap v269: 97 pgs: 97 active+clean; 353 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 465 op/s; 762 KiB/s, 30 keys/s, 16 objects/s recovering 2024-09-17T10:57:31.326 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:31 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:31.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:31 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:57:31.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:31 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:31.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:31 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T10:57:31.327 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:31 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:57:31.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 254824 9.32 MB/sec execute 563 sec latency 48.348 ms 2024-09-17T10:57:32.152 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:32 smithi006 ceph-mon[63965]: Upgrade: Updating osd.7 2024-09-17T10:57:32.153 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:32 smithi006 ceph-mon[63965]: Deploying daemon osd.7 on smithi149 2024-09-17T10:57:32.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:32 smithi149 ceph-mon[49697]: Upgrade: Updating osd.7 2024-09-17T10:57:32.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:32 smithi149 ceph-mon[49697]: Deploying daemon osd.7 on smithi149 2024-09-17T10:57:32.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 255110 9.32 MB/sec execute 564 sec latency 72.757 ms 2024-09-17T10:57:33.148 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:33 smithi149 ceph-mon[49697]: pgmap v270: 97 pgs: 97 active+clean; 347 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 420 op/s; 611 KiB/s, 24 keys/s, 12 objects/s recovering 2024-09-17T10:57:33.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:33 smithi006 ceph-mon[63965]: pgmap v270: 97 pgs: 97 active+clean; 347 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 420 op/s; 611 KiB/s, 24 keys/s, 12 objects/s recovering 2024-09-17T10:57:33.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 255362 9.31 MB/sec execute 565 sec latency 64.772 ms 2024-09-17T10:57:34.193 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:34 smithi006 ceph-mon[63965]: pgmap v271: 97 pgs: 97 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 516 op/s; 446 KiB/s, 6 objects/s recovering 2024-09-17T10:57:34.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:34 smithi149 ceph-mon[49697]: pgmap v271: 97 pgs: 97 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 516 op/s; 446 KiB/s, 6 objects/s recovering 2024-09-17T10:57:34.706 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:34 smithi149 systemd[1]: Stopping Ceph osd.7 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:57:34.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 255667 9.31 MB/sec execute 566 sec latency 66.985 ms 2024-09-17T10:57:35.060 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:34 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[39248]: 2024-09-17T10:57:34.766+0000 7fd1fe0fa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T10:57:35.060 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:34 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[39248]: 2024-09-17T10:57:34.766+0000 7fd1fe0fa700 -1 osd.7 143 *** Got signal Terminated *** 2024-09-17T10:57:35.060 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:34 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[39248]: 2024-09-17T10:57:34.766+0000 7fd1fe0fa700 -1 osd.7 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T10:57:35.232 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:35 smithi006 ceph-mon[63965]: osd.7 marked itself down and dead 2024-09-17T10:57:35.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:35 smithi149 ceph-mon[49697]: osd.7 marked itself down and dead 2024-09-17T10:57:35.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 255729 9.30 MB/sec execute 567 sec latency 830.719 ms 2024-09-17T10:57:36.043 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:35 smithi149 podman[59171]: 2024-09-17 10:57:35.754927819 +0000 UTC m=+1.109138166 container died 35f4f08ff1ae9d5fd4383e3f076eb95f277a9c418f51cf5077d32991fe6b0d5d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-type=git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, version=8) 2024-09-17T10:57:36.318 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:36 smithi006 ceph-mon[63965]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:57:36.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:36 smithi006 ceph-mon[63965]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T10:57:36.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:36 smithi006 ceph-mon[63965]: osdmap e144: 8 total, 7 up, 8 in 2024-09-17T10:57:36.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:36.319 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:36 smithi006 ceph-mon[63965]: pgmap v273: 97 pgs: 12 stale+active+clean, 85 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 445 op/s; 489 KiB/s, 3 objects/s recovering 2024-09-17T10:57:36.403 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:36 smithi149 ceph-mon[49697]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T10:57:36.403 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:36 smithi149 ceph-mon[49697]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T10:57:36.403 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:36 smithi149 ceph-mon[49697]: osdmap e144: 8 total, 7 up, 8 in 2024-09-17T10:57:36.403 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:36.403 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:36 smithi149 ceph-mon[49697]: pgmap v273: 97 pgs: 12 stale+active+clean, 85 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 445 op/s; 489 KiB/s, 3 objects/s recovering 2024-09-17T10:57:36.404 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:36 smithi149 podman[59171]: 2024-09-17 10:57:36.142065429 +0000 UTC m=+1.496275769 container cleanup 35f4f08ff1ae9d5fd4383e3f076eb95f277a9c418f51cf5077d32991fe6b0d5d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, maintainer=Guillaume Abrioux , name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, distribution-scope=public, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-type=git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container) 2024-09-17T10:57:36.404 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:36 smithi149 bash[59171]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7 2024-09-17T10:57:36.706 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:36 smithi149 podman[59187]: 2024-09-17 10:57:36.402396115 +0000 UTC m=+0.641787235 container remove 35f4f08ff1ae9d5fd4383e3f076eb95f277a9c418f51cf5077d32991fe6b0d5d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, release=754, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, distribution-scope=public, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc.) 2024-09-17T10:57:36.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 255929 9.30 MB/sec execute 568 sec latency 1100.041 ms 2024-09-17T10:57:37.091 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:36 smithi149 podman[59272]: 2024-09-17 10:57:36.721934458 +0000 UTC m=+0.023913661 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:57:37.091 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:36 smithi149 podman[59272]: 2024-09-17 10:57:36.845136237 +0000 UTC m=+0.147115426 container create a1ad0fd41e62f89bfae3ab311505b72149de5704e760e7febcdf08186440161a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-17T10:57:37.091 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:37 smithi149 podman[59272]: 2024-09-17 10:57:37.090002835 +0000 UTC m=+0.391982026 container init a1ad0fd41e62f89bfae3ab311505b72149de5704e760e7febcdf08186440161a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T10:57:37.341 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:37 smithi006 ceph-mon[63965]: osdmap e145: 8 total, 7 up, 8 in 2024-09-17T10:57:37.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:37 smithi149 ceph-mon[49697]: osdmap e145: 8 total, 7 up, 8 in 2024-09-17T10:57:37.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:37 smithi149 podman[59272]: 2024-09-17 10:57:37.094715978 +0000 UTC m=+0.396695171 container start a1ad0fd41e62f89bfae3ab311505b72149de5704e760e7febcdf08186440161a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-17T10:57:37.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:37 smithi149 podman[59272]: 2024-09-17 10:57:37.198171863 +0000 UTC m=+0.500151065 container attach a1ad0fd41e62f89bfae3ab311505b72149de5704e760e7febcdf08186440161a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-7498493) 2024-09-17T10:57:37.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:37 smithi149 podman[59272]: 2024-09-17 10:57:37.340039736 +0000 UTC m=+0.642018923 container died a1ad0fd41e62f89bfae3ab311505b72149de5704e760e7febcdf08186440161a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T10:57:37.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 256134 9.29 MB/sec execute 569 sec latency 162.628 ms 2024-09-17T10:57:38.111 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:38 smithi006 ceph-mon[63965]: pgmap v275: 97 pgs: 16 peering, 8 active+undersized+degraded, 73 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.8 MiB/s wr, 372 op/s; 27/1584 objects degraded (1.705%) 2024-09-17T10:57:38.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:38 smithi149 ceph-mon[49697]: pgmap v275: 97 pgs: 16 peering, 8 active+undersized+degraded, 73 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.8 MiB/s wr, 372 op/s; 27/1584 objects degraded (1.705%) 2024-09-17T10:57:38.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 256385 9.29 MB/sec execute 570 sec latency 65.900 ms 2024-09-17T10:57:39.130 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:39 smithi006 ceph-mon[63965]: Health check failed: Degraded data redundancy: 27/1584 objects degraded (1.705%), 8 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:39.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:39 smithi149 ceph-mon[49697]: Health check failed: Degraded data redundancy: 27/1584 objects degraded (1.705%), 8 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:39.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 podman[59294]: 2024-09-17 10:57:39.279947351 +0000 UTC m=+1.933639579 container remove a1ad0fd41e62f89bfae3ab311505b72149de5704e760e7febcdf08186440161a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-17T10:57:39.457 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-17T10:57:39.457 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 systemd[1]: Stopped Ceph osd.7 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:57:39.457 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 systemd[1]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 20.540s CPU time. 2024-09-17T10:57:39.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 256590 9.29 MB/sec execute 571 sec latency 67.094 ms 2024-09-17T10:57:39.869 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 systemd[1]: Starting Ceph osd.7 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T10:57:39.869 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 podman[59392]: 2024-09-17 10:57:39.6348181 +0000 UTC m=+0.017243241 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:57:39.869 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 podman[59392]: 2024-09-17 10:57:39.747905579 +0000 UTC m=+0.130330721 container create d02821d49c8403b935517b65077ba02281ed8a9432e1f4caaadcae9b60bc6e84 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-7498493, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:57:40.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:40 smithi006 ceph-mon[63965]: pgmap v276: 97 pgs: 3 active+undersized, 16 peering, 23 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.8 MiB/s wr, 386 op/s; 169/1608 objects degraded (10.510%) 2024-09-17T10:57:40.145 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:40 smithi149 ceph-mon[49697]: pgmap v276: 97 pgs: 3 active+undersized, 16 peering, 23 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.8 MiB/s wr, 386 op/s; 169/1608 objects degraded (10.510%) 2024-09-17T10:57:40.148 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 podman[59392]: 2024-09-17 10:57:39.984174404 +0000 UTC m=+0.366599565 container init d02821d49c8403b935517b65077ba02281ed8a9432e1f4caaadcae9b60bc6e84 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-17T10:57:40.148 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:39 smithi149 podman[59392]: 2024-09-17 10:57:39.989835148 +0000 UTC m=+0.372260298 container start d02821d49c8403b935517b65077ba02281ed8a9432e1f4caaadcae9b60bc6e84 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=main-7498493) 2024-09-17T10:57:40.148 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 podman[59392]: 2024-09-17 10:57:40.084280227 +0000 UTC m=+0.466705371 container attach d02821d49c8403b935517b65077ba02281ed8a9432e1f4caaadcae9b60bc6e84 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T10:57:40.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:57:40.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 bash[59392]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:57:40.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:57:40.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 bash[59392]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T10:57:40.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 256826 9.28 MB/sec execute 572 sec latency 59.415 ms 2024-09-17T10:57:41.022 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T10:57:41.023 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 bash[59392]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T10:57:41.023 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T10:57:41.023 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:40 smithi149 bash[59392]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T10:57:41.381 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T10:57:41.381 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 bash[59392]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T10:57:41.381 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T10:57:41.381 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 bash[59392]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T10:57:41.381 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T10:57:41.382 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 bash[59392]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T10:57:41.382 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T10:57:41.382 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 bash[59392]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T10:57:41.382 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate[59407]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T10:57:41.382 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 bash[59392]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T10:57:41.382 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 podman[59392]: 2024-09-17 10:57:41.071532596 +0000 UTC m=+1.453957745 container died d02821d49c8403b935517b65077ba02281ed8a9432e1f4caaadcae9b60bc6e84 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T10:57:41.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 257077 9.27 MB/sec execute 573 sec latency 60.850 ms 2024-09-17T10:57:41.949 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 podman[59547]: 2024-09-17 10:57:41.697797694 +0000 UTC m=+0.619433536 container remove d02821d49c8403b935517b65077ba02281ed8a9432e1f4caaadcae9b60bc6e84 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=main-7498493, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T10:57:41.949 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 podman[59602]: 2024-09-17 10:57:41.845043343 +0000 UTC m=+0.020764883 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T10:57:41.950 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:41 smithi149 podman[59602]: 2024-09-17 10:57:41.948768731 +0000 UTC m=+0.124490261 container create 8f7c760b4de3c7de3dec920d445c5487214b3dc1bcdcf2034055009f2491e591 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, RELEASE=main-7498493, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T10:57:42.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:42 smithi149 podman[59602]: 2024-09-17 10:57:42.17632247 +0000 UTC m=+0.352044014 container init 8f7c760b4de3c7de3dec920d445c5487214b3dc1bcdcf2034055009f2491e591 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T10:57:42.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:42 smithi149 podman[59602]: 2024-09-17 10:57:42.180353361 +0000 UTC m=+0.356074905 container start 8f7c760b4de3c7de3dec920d445c5487214b3dc1bcdcf2034055009f2491e591 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T10:57:42.620 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:42 smithi149 ceph-osd[59620]: -- 172.21.15.149:0/3344016057 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ea7495a1e0 con 0x55ea748e6000 2024-09-17T10:57:42.620 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:42 smithi149 bash[59602]: 8f7c760b4de3c7de3dec920d445c5487214b3dc1bcdcf2034055009f2491e591 2024-09-17T10:57:42.620 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:42 smithi149 systemd[1]: Started Ceph osd.7 for ddf74116-74e0-11ef-bceb-c7b262605968. 2024-09-17T10:57:42.704 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:42 smithi006 ceph-mon[63965]: pgmap v277: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 304 op/s; 250/1620 objects degraded (15.432%); 258 B/s, 0 objects/s recovering 2024-09-17T10:57:42.704 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:42.704 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:42.704 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:42 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:57:42.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 257256 9.26 MB/sec execute 574 sec latency 53.845 ms 2024-09-17T10:57:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:42 smithi149 ceph-mon[49697]: pgmap v277: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 304 op/s; 250/1620 objects degraded (15.432%); 258 B/s, 0 objects/s recovering 2024-09-17T10:57:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:42 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:57:43.754 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:43 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 254/1656 objects degraded (15.338%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:43.851 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:43 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 254/1656 objects degraded (15.338%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:43.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 257432 9.25 MB/sec execute 575 sec latency 59.954 ms 2024-09-17T10:57:44.176 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:43 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T10:57:43.849+0000 7f9f277ec640 -1 Falling back to public interface 2024-09-17T10:57:44.777 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:44 smithi006 ceph-mon[63965]: pgmap v278: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.8 MiB/s wr, 323 op/s; 254/1656 objects degraded (15.338%); 243 B/s, 0 objects/s recovering 2024-09-17T10:57:44.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 257724 9.25 MB/sec execute 576 sec latency 73.096 ms 2024-09-17T10:57:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:44 smithi149 ceph-mon[49697]: pgmap v278: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.8 MiB/s wr, 323 op/s; 254/1656 objects degraded (15.338%); 243 B/s, 0 objects/s recovering 2024-09-17T10:57:45.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 258038 9.24 MB/sec execute 577 sec latency 50.244 ms 2024-09-17T10:57:46.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:46.062 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:46 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:57:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:46.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:46 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:57:46.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 258356 9.24 MB/sec execute 578 sec latency 64.215 ms 2024-09-17T10:57:47.091 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:47 smithi006 ceph-mon[63965]: pgmap v279: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.3 MiB/s wr, 275 op/s; 254/1656 objects degraded (15.338%); 206 B/s, 0 objects/s recovering 2024-09-17T10:57:47.092 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:47.092 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:47.092 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:47.092 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:47.272 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:47 smithi149 ceph-mon[49697]: pgmap v279: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.3 MiB/s wr, 275 op/s; 254/1656 objects degraded (15.338%); 206 B/s, 0 objects/s recovering 2024-09-17T10:57:47.272 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:47.272 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:47.272 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:47.272 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:47.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 258680 9.25 MB/sec execute 579 sec latency 80.502 ms 2024-09-17T10:57:48.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 258892 9.24 MB/sec execute 580 sec latency 55.165 ms 2024-09-17T10:57:49.180 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:49 smithi006 ceph-mon[63965]: pgmap v280: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.3 MiB/s wr, 338 op/s; 249/1623 objects degraded (15.342%); 179 B/s, 0 objects/s recovering 2024-09-17T10:57:49.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:49 smithi149 ceph-mon[49697]: pgmap v280: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.3 MiB/s wr, 338 op/s; 249/1623 objects degraded (15.342%); 179 B/s, 0 objects/s recovering 2024-09-17T10:57:49.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 259145 9.23 MB/sec execute 581 sec latency 75.823 ms 2024-09-17T10:57:50.064 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:57:50.205 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:50 smithi006 ceph-mon[63965]: pgmap v281: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.6 MiB/s wr, 346 op/s; 250/1620 objects degraded (15.432%); 172 B/s, 0 objects/s recovering 2024-09-17T10:57:50.206 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:50 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 250/1620 objects degraded (15.432%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:50 smithi149 ceph-mon[49697]: pgmap v281: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.6 MiB/s wr, 346 op/s; 250/1620 objects degraded (15.432%); 172 B/s, 0 objects/s recovering 2024-09-17T10:57:50.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:50 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 250/1620 objects degraded (15.432%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:50.854 INFO:teuthology.orchestra.run.smithi006.stdout: 5 259443 9.23 MB/sec execute 582 sec latency 61.001 ms 2024-09-17T10:57:51.313 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:57:51.314 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (9m) 2m ago 18m 18.0M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:57:51.314 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (5m) 2m ago 18m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:57:51.314 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (5m) 5s ago 17m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:57:51.314 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (8m) 2m ago 18m 42.7M - dad864ee21e9 89df198e1a9b 2024-09-17T10:57:51.314 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (12m) 2m ago 12m 38.0M - 17.2.0 e1d6a67b021e 0b85ce757448 2024-09-17T10:57:51.314 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (12m) 5s ago 12m 16.0M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:57:51.315 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (8m) 2m ago 19m 518M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:57:51.315 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (7m) 5s ago 17m 457M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:57:51.315 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (6m) 2m ago 19m 52.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:57:51.315 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (6m) 5s ago 17m 45.3M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:57:51.315 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (11m) 2m ago 11m 286M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:57:51.315 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (11m) 5s ago 11m 88.6M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:57:51.315 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (8m) 2m ago 17m 9382k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:57:51.316 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (8m) 5s ago 17m 11.9M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:57:51.316 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (5m) 2m ago 16m 334M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:57:51.316 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (4m) 2m ago 16m 254M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:57:51.316 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (3m) 2m ago 15m 213M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:57:51.316 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (2m) 2m ago 15m 11.7M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T10:57:51.316 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (119s) 5s ago 14m 328M 4096M 19.3.0-4937-g74984932 6c1fcab85602 016956823ff1 2024-09-17T10:57:51.317 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (82s) 5s ago 14m 289M 4096M 19.3.0-4937-g74984932 6c1fcab85602 144416f89b4a 2024-09-17T10:57:51.317 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (46s) 5s ago 14m 244M 4096M 19.3.0-4937-g74984932 6c1fcab85602 2908fd9a4510 2024-09-17T10:57:51.317 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (9s) 5s ago 13m 11.1M 4096M 19.3.0-4937-g74984932 6c1fcab85602 8f7c760b4de3 2024-09-17T10:57:51.317 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (7m) 2m ago 17m 53.3M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:57:51.528 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:51.528 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.528 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.528 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all osd 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-17T10:57:51.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T10:57:51.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-17T10:57:51.531 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T10:57:51.531 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T10:57:51.531 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:51 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-17T10:57:51.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:51.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all osd 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-17T10:57:51.708 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T10:57:51.709 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-17T10:57:51.710 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T10:57:51.710 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T10:57:51.710 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:51 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-17T10:57:51.855 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:57:51.855 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:57:51.855 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:57:51.855 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:57:51.855 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:57:51.855 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:57:51.855 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:57:51.855 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 7 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 11 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:57:51.856 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:57:51.857 INFO:teuthology.orchestra.run.smithi006.stdout: 5 259635 9.23 MB/sec execute 583 sec latency 75.855 ms 2024-09-17T10:57:52.308 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:52 smithi006 ceph-mon[63965]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:52.308 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:52 smithi006 ceph-mon[63965]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 420 op/s; 258/1638 objects degraded (15.751%); 172 B/s, 0 objects/s recovering 2024-09-17T10:57:52.308 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:52 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2610529350' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:52.308 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:52 smithi006 ceph-mon[63965]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T10:57:52.308 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T10:57:52.308 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:52 smithi006 ceph-mon[63965]: osdmap e146: 8 total, 7 up, 8 in 2024-09-17T10:57:52.308 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi006.nripkg"]} : dispatch 2024-09-17T10:57:52.308 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:52 smithi006 ceph-mon[63965]: Upgrade: It appears safe to stop mds.foofs.smithi006.nripkg 2024-09-17T10:57:52.360 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:57:52.360 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:57:52.360 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:57:52.360 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: "osd", 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "14/23 daemons upgraded", 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mds daemons", 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:57:52.361 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:57:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:52 smithi149 ceph-mon[49697]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:52 smithi149 ceph-mon[49697]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 420 op/s; 258/1638 objects degraded (15.751%); 172 B/s, 0 objects/s recovering 2024-09-17T10:57:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:52 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2610529350' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:57:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:52 smithi149 ceph-mon[49697]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T10:57:52.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T10:57:52.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:52 smithi149 ceph-mon[49697]: osdmap e146: 8 total, 7 up, 8 in 2024-09-17T10:57:52.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi006.nripkg"]} : dispatch 2024-09-17T10:57:52.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:52 smithi149 ceph-mon[49697]: Upgrade: It appears safe to stop mds.foofs.smithi006.nripkg 2024-09-17T10:57:52.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 259858 9.23 MB/sec execute 584 sec latency 71.292 ms 2024-09-17T10:57:52.949 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 258/1638 objects degraded (15.751%), 35 pgs degraded 2024-09-17T10:57:52.949 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: osd.7 (root=default,host=smithi149) is down 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 258/1638 objects degraded (15.751%), 35 pgs degraded 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-09-17T10:57:52.950 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-09-17T10:57:52.951 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-09-17T10:57:52.952 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-09-17T10:57:52.953 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-09-17T10:57:52.953 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-09-17T10:57:52.953 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-09-17T10:57:53.375 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:53 smithi006 ceph-mon[63965]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:53.376 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:53 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1901520508' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:57:53.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:53 smithi149 ceph-mon[49697]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:57:53.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:53 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/1901520508' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:57:53.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260119 9.23 MB/sec execute 585 sec latency 52.956 ms 2024-09-17T10:57:54.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:53 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T10:57:53.853+0000 7f9f277ec640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-17T10:57:54.403 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:54 smithi006 ceph-mon[63965]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 388 op/s; 255/1626 objects degraded (15.683%) 2024-09-17T10:57:54.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:54 smithi149 ceph-mon[49697]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 388 op/s; 255/1626 objects degraded (15.683%) 2024-09-17T10:57:54.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260456 9.22 MB/sec execute 586 sec latency 39.476 ms 2024-09-17T10:57:55.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260712 9.22 MB/sec execute 587 sec latency 54.451 ms 2024-09-17T10:57:55.921 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:55 smithi006 ceph-mon[63965]: Upgrade: Updating mds.foofs.smithi006.nripkg 2024-09-17T10:57:55.922 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:55.922 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.nripkg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T10:57:55.922 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:55 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:57:55.922 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:55 smithi006 ceph-mon[63965]: Deploying daemon mds.foofs.smithi006.nripkg on smithi006 2024-09-17T10:57:55.922 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:55 smithi006 ceph-mon[63965]: from='osd.7 [v2:172.21.15.149:6824/2426671602,v1:172.21.15.149:6825/2426671602]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T10:57:55.922 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:55 smithi006 ceph-mon[63965]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T10:57:55.956 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T10:57:55.516+0000 7f9f277ec640 -1 osd.7 143 log_to_monitors true 2024-09-17T10:57:55.956 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T10:57:55.742+0000 7f9f1f596640 -1 osd.7 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T10:57:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-mon[49697]: Upgrade: Updating mds.foofs.smithi006.nripkg 2024-09-17T10:57:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:57:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.nripkg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T10:57:55.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:57:55.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-mon[49697]: Deploying daemon mds.foofs.smithi006.nripkg on smithi006 2024-09-17T10:57:55.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-mon[49697]: from='osd.7 [v2:172.21.15.149:6824/2426671602,v1:172.21.15.149:6825/2426671602]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T10:57:55.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:55 smithi149 ceph-mon[49697]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T10:57:56.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260932 9.21 MB/sec execute 588 sec latency 152.179 ms 2024-09-17T10:57:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:56 smithi006 ceph-mon[63965]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 388 op/s; 255/1626 objects degraded (15.683%) 2024-09-17T10:57:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:56 smithi006 ceph-mon[63965]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T10:57:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:56 smithi006 ceph-mon[63965]: osdmap e147: 8 total, 7 up, 8 in 2024-09-17T10:57:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:56 smithi006 ceph-mon[63965]: from='osd.7 [v2:172.21.15.149:6824/2426671602,v1:172.21.15.149:6825/2426671602]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:57:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:56 smithi006 ceph-mon[63965]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:57:57.090 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:56 smithi149 ceph-mon[49697]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 388 op/s; 255/1626 objects degraded (15.683%) 2024-09-17T10:57:57.090 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:56 smithi149 ceph-mon[49697]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T10:57:57.090 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:56 smithi149 ceph-mon[49697]: osdmap e147: 8 total, 7 up, 8 in 2024-09-17T10:57:57.090 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:56 smithi149 ceph-mon[49697]: from='osd.7 [v2:172.21.15.149:6824/2426671602,v1:172.21.15.149:6825/2426671602]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:57:57.090 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:56 smithi149 ceph-mon[49697]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]} : dispatch 2024-09-17T10:57:57.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261010 9.19 MB/sec execute 589 sec latency 1148.641 ms 2024-09-17T10:57:57.916 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:57 smithi006 ceph-mon[63965]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:57:57.916 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:57 smithi006 ceph-mon[63965]: osd.7 [v2:172.21.15.149:6824/2426671602,v1:172.21.15.149:6825/2426671602] boot 2024-09-17T10:57:57.917 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:57 smithi006 ceph-mon[63965]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T10:57:57.917 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:57 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T10:57:57.917 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:57 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 255/1626 objects degraded (15.683%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:57.917 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:57 smithi006 ceph-mon[63965]: osdmap e149: 8 total, 8 up, 8 in 2024-09-17T10:57:58.105 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:57 smithi149 ceph-mon[49697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T10:57:58.105 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:57 smithi149 ceph-mon[49697]: osd.7 [v2:172.21.15.149:6824/2426671602,v1:172.21.15.149:6825/2426671602] boot 2024-09-17T10:57:58.105 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:57 smithi149 ceph-mon[49697]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T10:57:58.105 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:57 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T10:57:58.106 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:57 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 255/1626 objects degraded (15.683%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T10:57:58.106 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:57 smithi149 ceph-mon[49697]: osdmap e149: 8 total, 8 up, 8 in 2024-09-17T10:57:58.737 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[63961]: 2024-09-17T10:57:58.473+0000 7f8897eac640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:57:58.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.18 MB/sec execute 590 sec latency 1808.719 ms 2024-09-17T10:57:58.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:57:58.552+0000 7f48a2209640 -1 bdev(0x5641b61fec00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-17T10:57:58.957 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:57:58.552+0000 7f48a3a0c640 -1 bdev(0x5641b61fec00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-17T10:57:58.957 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:57:58.557+0000 7f489ea02640 -1 bdev(0x5641b61fec00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 6 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: pgmap v289: 97 pgs: 7 peering, 7 active+undersized, 28 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.1 MiB/s wr, 376 op/s; 237/1680 objects degraded (14.107%) 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: osdmap e150: 8 total, 8 up, 8 in 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: osdmap e151: 8 total, 8 up, 8 in 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: Standby daemon mds.foofs.smithi149.hmuwiq assigned to filesystem foofs as rank 0 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T10:57:58.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T10:57:58.958 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:57:58 smithi149 ceph-mon[49697]: fsmap foofs:1/1 {0=foofs.smithi149.hmuwiq=up:replay} 2024-09-17T10:57:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: pgmap v289: 97 pgs: 7 peering, 7 active+undersized, 28 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.1 MiB/s wr, 376 op/s; 237/1680 objects degraded (14.107%) 2024-09-17T10:57:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: osdmap e150: 8 total, 8 up, 8 in 2024-09-17T10:57:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T10:57:59.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:57:59.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: osdmap e151: 8 total, 8 up, 8 in 2024-09-17T10:57:59.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: Standby daemon mds.foofs.smithi149.hmuwiq assigned to filesystem foofs as rank 0 2024-09-17T10:57:59.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T10:57:59.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T10:57:59.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T10:57:59.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:57:58 smithi006 ceph-mon[63965]: fsmap foofs:1/1 {0=foofs.smithi149.hmuwiq=up:replay} 2024-09-17T10:57:59.855 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.16 MB/sec execute 591 sec latency 1339.979 ms 2024-09-17T10:58:00.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.15 MB/sec execute 592 sec latency 2340.030 ms 2024-09-17T10:58:00.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:00 smithi006 ceph-mon[63965]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T10:58:00.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:00 smithi006 ceph-mon[63965]: pgmap v293: 97 pgs: 5 remapped+peering, 7 peering, 4 active+undersized, 18 active+undersized+degraded, 63 active+clean; 348 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 MiB/s wr, 119 op/s; 132/1701 objects degraded (7.760%); 165 KiB/s, 12 objects/s recovering 2024-09-17T10:58:00.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:00.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:58:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:00 smithi149 ceph-mon[49697]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T10:58:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:00 smithi149 ceph-mon[49697]: pgmap v293: 97 pgs: 5 remapped+peering, 7 peering, 4 active+undersized, 18 active+undersized+degraded, 63 active+clean; 348 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 MiB/s wr, 119 op/s; 132/1701 objects degraded (7.760%); 165 KiB/s, 12 objects/s recovering 2024-09-17T10:58:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:58:01.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.13 MB/sec execute 593 sec latency 3340.316 ms 2024-09-17T10:58:01.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:01 smithi006 ceph-mon[63965]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T10:58:01.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:01 smithi006 ceph-mon[63965]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T10:58:01.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:01 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.6:6834/4158874117,v1:172.21.15.6:6835/4158874117] up:boot 2024-09-17T10:58:01.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:01 smithi006 ceph-mon[63965]: fsmap foofs:1/1 {0=foofs.smithi149.hmuwiq=up:replay} 1 up:standby 2024-09-17T10:58:01.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:01 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"} : dispatch 2024-09-17T10:58:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:01 smithi149 ceph-mon[49697]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T10:58:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:01 smithi149 ceph-mon[49697]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T10:58:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:01 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.6:6834/4158874117,v1:172.21.15.6:6835/4158874117] up:boot 2024-09-17T10:58:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:01 smithi149 ceph-mon[49697]: fsmap foofs:1/1 {0=foofs.smithi149.hmuwiq=up:replay} 1 up:standby 2024-09-17T10:58:01.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:01 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"} : dispatch 2024-09-17T10:58:02.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.12 MB/sec execute 594 sec latency 4340.362 ms 2024-09-17T10:58:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:02 smithi006 ceph-mon[63965]: pgmap v295: 97 pgs: 2 remapped+peering, 7 peering, 88 active+clean; 348 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 146 op/s; 3.2 MiB/s, 55 objects/s recovering 2024-09-17T10:58:02.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:02 smithi006 ceph-mon[63965]: Health check update: Degraded data redundancy: 132/1701 objects degraded (7.760%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T10:58:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:02 smithi149 ceph-mon[49697]: pgmap v295: 97 pgs: 2 remapped+peering, 7 peering, 88 active+clean; 348 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 146 op/s; 3.2 MiB/s, 55 objects/s recovering 2024-09-17T10:58:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:02 smithi149 ceph-mon[49697]: Health check update: Degraded data redundancy: 132/1701 objects degraded (7.760%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T10:58:03.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.10 MB/sec execute 595 sec latency 5340.390 ms 2024-09-17T10:58:03.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:03 smithi006 ceph-mon[63965]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 132/1701 objects degraded (7.760%), 18 pgs degraded) 2024-09-17T10:58:03.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:03 smithi149 ceph-mon[49697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 132/1701 objects degraded (7.760%), 18 pgs degraded) 2024-09-17T10:58:04.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.09 MB/sec execute 596 sec latency 6340.438 ms 2024-09-17T10:58:04.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:04 smithi006 ceph-mon[63965]: reconnect by client.24347 172.21.15.149:0/3560729512 after 0.000999981 2024-09-17T10:58:04.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:04 smithi006 ceph-mon[63965]: reconnect by client.14556 172.21.15.6:0/116617968 after 0.000999981 2024-09-17T10:58:04.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:04 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] up:reconnect 2024-09-17T10:58:04.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:04 smithi006 ceph-mon[63965]: fsmap foofs:1/1 {0=foofs.smithi149.hmuwiq=up:reconnect} 1 up:standby 2024-09-17T10:58:04.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:04 smithi006 ceph-mon[63965]: pgmap v296: 97 pgs: 97 active+clean; 349 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 757 KiB/s wr, 100 op/s; 2.7 MiB/s, 43 objects/s recovering 2024-09-17T10:58:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:04 smithi149 ceph-mon[49697]: reconnect by client.24347 172.21.15.149:0/3560729512 after 0.000999981 2024-09-17T10:58:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:04 smithi149 ceph-mon[49697]: reconnect by client.14556 172.21.15.6:0/116617968 after 0.000999981 2024-09-17T10:58:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:04 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] up:reconnect 2024-09-17T10:58:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:04 smithi149 ceph-mon[49697]: fsmap foofs:1/1 {0=foofs.smithi149.hmuwiq=up:reconnect} 1 up:standby 2024-09-17T10:58:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:04 smithi149 ceph-mon[49697]: pgmap v296: 97 pgs: 97 active+clean; 349 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 757 KiB/s wr, 100 op/s; 2.7 MiB/s, 43 objects/s recovering 2024-09-17T10:58:05.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.07 MB/sec execute 597 sec latency 7340.500 ms 2024-09-17T10:58:05.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:05 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] up:rejoin 2024-09-17T10:58:05.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:05 smithi006 ceph-mon[63965]: fsmap foofs:1/1 {0=foofs.smithi149.hmuwiq=up:rejoin} 1 up:standby 2024-09-17T10:58:05.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:05 smithi006 ceph-mon[63965]: daemon mds.foofs.smithi149.hmuwiq is now active in filesystem foofs as rank 0 2024-09-17T10:58:05.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:05 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] up:rejoin 2024-09-17T10:58:05.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:05 smithi149 ceph-mon[49697]: fsmap foofs:1/1 {0=foofs.smithi149.hmuwiq=up:rejoin} 1 up:standby 2024-09-17T10:58:05.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:05 smithi149 ceph-mon[49697]: daemon mds.foofs.smithi149.hmuwiq is now active in filesystem foofs as rank 0 2024-09-17T10:58:06.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.06 MB/sec execute 598 sec latency 8340.557 ms 2024-09-17T10:58:06.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:06 smithi006 ceph-mon[63965]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T10:58:06.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:06 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:58:06.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:06 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] up:active 2024-09-17T10:58:06.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:06 smithi006 ceph-mon[63965]: fsmap foofs:1 {0=foofs.smithi149.hmuwiq=up:active} 1 up:standby 2024-09-17T10:58:06.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:06 smithi006 ceph-mon[63965]: pgmap v297: 97 pgs: 97 active+clean; 349 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 565 KiB/s wr, 74 op/s; 2.0 MiB/s, 32 objects/s recovering 2024-09-17T10:58:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:06 smithi149 ceph-mon[49697]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T10:58:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:06 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:58:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:06 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.149:6832/3630302740,v1:172.21.15.149:6833/3630302740] up:active 2024-09-17T10:58:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:06 smithi149 ceph-mon[49697]: fsmap foofs:1 {0=foofs.smithi149.hmuwiq=up:active} 1 up:standby 2024-09-17T10:58:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:06 smithi149 ceph-mon[49697]: pgmap v297: 97 pgs: 97 active+clean; 349 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 565 KiB/s wr, 74 op/s; 2.0 MiB/s, 32 objects/s recovering 2024-09-17T10:58:07.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261012 9.04 MB/sec execute 599 sec latency 9340.604 ms 2024-09-17T10:58:08.753 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:08 smithi006 ceph-mon[63965]: pgmap v298: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 498 KiB/s wr, 83 op/s; 1.8 MiB/s, 28 objects/s recovering 2024-09-17T10:58:08.856 INFO:teuthology.orchestra.run.smithi006.stdout: 5 cleanup 600 sec 2024-09-17T10:58:08.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:08 smithi149 ceph-mon[49697]: pgmap v298: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 498 KiB/s wr, 83 op/s; 1.8 MiB/s, 28 objects/s recovering 2024-09-17T10:58:09.122 INFO:teuthology.orchestra.run.smithi006.stdout: 0 cleanup 600 sec 2024-09-17T10:58:09.122 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:58:09.122 INFO:teuthology.orchestra.run.smithi006.stdout: Operation Count AvgLat MaxLat 2024-09-17T10:58:09.122 INFO:teuthology.orchestra.run.smithi006.stdout: ---------------------------------------- 2024-09-17T10:58:09.122 INFO:teuthology.orchestra.run.smithi006.stdout: NTCreateX 174233 4.865 1096.862 2024-09-17T10:58:09.122 INFO:teuthology.orchestra.run.smithi006.stdout: Close 127815 0.147 23.586 2024-09-17T10:58:09.122 INFO:teuthology.orchestra.run.smithi006.stdout: Rename 7373 10.291 1103.782 2024-09-17T10:58:09.122 INFO:teuthology.orchestra.run.smithi006.stdout: Unlink 35311 7.010 1059.868 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: Qpathinfo 157843 1.268 1045.242 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: Qfileinfo 27462 0.007 2.786 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: Qfsinfo 28982 0.659 37.700 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: Sfileinfo 14191 14.901 9474.299 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: Find 60955 1.056 30.132 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: WriteX 85970 15.215 9484.539 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: ReadX 272684 0.011 10.806 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: LockX 566 0.406 11.811 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: UnlockX 566 0.363 3.555 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: Flush 12227 0.003 0.046 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:58:09.123 INFO:teuthology.orchestra.run.smithi006.stdout:Throughput 9.04115 MB/sec 5 clients 5 procs max_latency=9340.604 ms 2024-09-17T10:58:09.125 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-17T10:58:09.183 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-17T10:58:09.340 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-17T10:58:09.385 INFO:teuthology.orchestra.run.smithi006.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T10:58:09.386 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:58:09.386 INFO:teuthology.orchestra.run.smithi006.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-17T10:58:09.386 INFO:teuthology.orchestra.run.smithi006.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-17T10:58:09.486 INFO:teuthology.orchestra.run.smithi006.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T10:58:09.487 INFO:teuthology.orchestra.run.smithi006.stdout:releasing clients 2024-09-17T10:58:10.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:10 smithi006 ceph-mon[63965]: pgmap v299: 97 pgs: 97 active+clean; 339 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 403 KiB/s wr, 74 op/s; 1.4 MiB/s, 23 objects/s recovering 2024-09-17T10:58:10.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:10 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:10 smithi149 ceph-mon[49697]: pgmap v299: 97 pgs: 97 active+clean; 339 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 403 KiB/s wr, 74 op/s; 1.4 MiB/s, 23 objects/s recovering 2024-09-17T10:58:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:10 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:11.487 INFO:teuthology.orchestra.run.smithi006.stdout: 5 374 25.75 MB/sec execute 1 sec latency 49.117 ms 2024-09-17T10:58:11.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:11 smithi006 ceph-mon[63965]: daemon mds.foofs.smithi006.nripkg restarted 2024-09-17T10:58:11.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:11 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:11.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:11 smithi006 ceph-mon[63965]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T10:58:11.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:11 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:11.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:11 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:11 smithi149 ceph-mon[49697]: daemon mds.foofs.smithi006.nripkg restarted 2024-09-17T10:58:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:11 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:11 smithi149 ceph-mon[49697]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T10:58:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:11 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:11.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:11 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:12.487 INFO:teuthology.orchestra.run.smithi006.stdout: 5 522 25.21 MB/sec execute 2 sec latency 62.987 ms 2024-09-17T10:58:12.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:12 smithi006 ceph-mon[63965]: pgmap v301: 97 pgs: 97 active+clean; 361 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.3 MiB/s wr, 66 op/s; 193 KiB/s, 4 objects/s recovering 2024-09-17T10:58:12.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:12 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771] up:boot 2024-09-17T10:58:12.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:12 smithi006 ceph-mon[63965]: fsmap foofs:1 {0=foofs.smithi149.hmuwiq=up:active} 1 up:standby 2024-09-17T10:58:12.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:12 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"} : dispatch 2024-09-17T10:58:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:12 smithi149 ceph-mon[49697]: pgmap v301: 97 pgs: 97 active+clean; 361 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.3 MiB/s wr, 66 op/s; 193 KiB/s, 4 objects/s recovering 2024-09-17T10:58:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:12 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771] up:boot 2024-09-17T10:58:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:12 smithi149 ceph-mon[49697]: fsmap foofs:1 {0=foofs.smithi149.hmuwiq=up:active} 1 up:standby 2024-09-17T10:58:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:12 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi006.nripkg"} : dispatch 2024-09-17T10:58:13.487 INFO:teuthology.orchestra.run.smithi006.stdout: 5 844 22.33 MB/sec execute 3 sec latency 37.095 ms 2024-09-17T10:58:14.487 INFO:teuthology.orchestra.run.smithi006.stdout: 5 1217 18.64 MB/sec execute 4 sec latency 26.905 ms 2024-09-17T10:58:14.788 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:14 smithi006 ceph-mon[63965]: pgmap v302: 97 pgs: 97 active+clean; 365 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.5 MiB/s wr, 182 op/s 2024-09-17T10:58:14.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:14 smithi149 ceph-mon[49697]: pgmap v302: 97 pgs: 97 active+clean; 365 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.5 MiB/s wr, 182 op/s 2024-09-17T10:58:15.487 INFO:teuthology.orchestra.run.smithi006.stdout: 5 cleanup 5 sec 2024-09-17T10:58:15.654 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b60700000029' failed - Device or resource busy 2024-09-17T10:58:15.655 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b5ef0000002a' failed - Device or resource busy 2024-09-17T10:58:15.655 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b60c00000032' failed - Device or resource busy 2024-09-17T10:58:15.656 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b5e000000025' failed - Device or resource busy 2024-09-17T10:58:15.657 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b5d600000027' failed - Device or resource busy 2024-09-17T10:58:15.657 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b5f40000002f' failed - Device or resource busy 2024-09-17T10:58:15.658 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b6020000002b' failed - Device or resource busy 2024-09-17T10:58:15.658 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b5fd00000026' failed - Device or resource busy 2024-09-17T10:58:15.705 INFO:teuthology.orchestra.run.smithi006.stdout:[1610] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b5ea00000028' failed - Device or resource busy 2024-09-17T10:58:15.789 INFO:teuthology.orchestra.run.smithi006.stdout: 0 cleanup 5 sec 2024-09-17T10:58:15.789 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:58:15.789 INFO:teuthology.orchestra.run.smithi006.stdout: Operation Count AvgLat MaxLat 2024-09-17T10:58:15.789 INFO:teuthology.orchestra.run.smithi006.stdout: ---------------------------------------- 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: NTCreateX 1119 6.102 62.980 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Close 944 6.148 45.469 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Rename 50 9.456 37.088 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Unlink 106 5.283 25.956 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Qpathinfo 723 1.664 24.155 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Qfileinfo 410 14.801 54.280 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Qfsinfo 165 0.530 3.367 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Sfileinfo 170 12.237 24.894 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Find 297 0.986 4.261 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: WriteX 1783 0.024 0.089 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: ReadX 1267 0.013 0.490 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: Flush 131 12.505 26.421 2024-09-17T10:58:15.790 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T10:58:15.791 INFO:teuthology.orchestra.run.smithi006.stdout:Throughput 18.6376 MB/sec 5 clients 5 procs max_latency=62.987 ms 2024-09-17T10:58:15.792 DEBUG:teuthology.parallel:result is None 2024-09-17T10:58:16.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:16.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:16 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:58:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:16.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:16 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:58:17.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:17 smithi006 ceph-mon[63965]: pgmap v303: 97 pgs: 97 active+clean; 365 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.5 MiB/s wr, 182 op/s 2024-09-17T10:58:17.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:17 smithi149 ceph-mon[49697]: pgmap v303: 97 pgs: 97 active+clean; 365 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.5 MiB/s wr, 182 op/s 2024-09-17T10:58:19.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:19 smithi006 ceph-mon[63965]: pgmap v304: 97 pgs: 97 active+clean; 349 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 12 MiB/s wr, 413 op/s 2024-09-17T10:58:19.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:19.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:19 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:19.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:19 smithi149 ceph-mon[49697]: pgmap v304: 97 pgs: 97 active+clean; 349 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 12 MiB/s wr, 413 op/s 2024-09-17T10:58:19.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:19.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:19 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:20.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:20 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:20.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:20 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:20.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:20 smithi006 ceph-mon[63965]: pgmap v305: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 12 MiB/s wr, 426 op/s 2024-09-17T10:58:20.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:20 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:20.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:20 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:20.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:20 smithi149 ceph-mon[49697]: pgmap v305: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 12 MiB/s wr, 426 op/s 2024-09-17T10:58:22.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:22 smithi006 ceph-mon[63965]: pgmap v306: 97 pgs: 97 active+clean; 278 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 9.8 MiB/s wr, 399 op/s 2024-09-17T10:58:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:22 smithi149 ceph-mon[49697]: pgmap v306: 97 pgs: 97 active+clean; 278 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 9.8 MiB/s wr, 399 op/s 2024-09-17T10:58:23.334 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:58:24.052 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:58:24.052 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (9m) 5s ago 18m 20.6M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:58:24.053 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (6m) 5s ago 18m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:58:24.053 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (6m) 37s ago 18m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:58:24.053 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (9m) 5s ago 18m 44.8M - dad864ee21e9 89df198e1a9b 2024-09-17T10:58:24.053 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (13s) 5s ago 12m 11.7M - 19.3.0-4937-g74984932 6c1fcab85602 7492dcce75c5 2024-09-17T10:58:24.053 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (12m) 37s ago 12m 16.0M - 17.2.0 e1d6a67b021e ea98be5f135d 2024-09-17T10:58:24.053 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (8m) 5s ago 20m 523M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:58:24.054 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (7m) 37s ago 18m 457M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:58:24.054 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (7m) 5s ago 20m 62.2M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:58:24.054 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (7m) 37s ago 18m 45.3M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:58:24.054 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (12m) 5s ago 12m 312M - 4.0 e1d6a67b021e 95f762031c1d 2024-09-17T10:58:24.054 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (12m) 37s ago 12m 88.6M - 4.0 e1d6a67b021e fe96979833b6 2024-09-17T10:58:24.054 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (9m) 5s ago 18m 9365k - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:58:24.055 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (8m) 37s ago 17m 11.9M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:58:24.055 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (5m) 5s ago 17m 389M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:58:24.055 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (4m) 5s ago 16m 393M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:58:24.055 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (4m) 5s ago 16m 370M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:58:24.055 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (3m) 5s ago 15m 380M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T10:58:24.055 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (2m) 37s ago 15m 328M 4096M 19.3.0-4937-g74984932 6c1fcab85602 016956823ff1 2024-09-17T10:58:24.056 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (114s) 37s ago 15m 289M 4096M 19.3.0-4937-g74984932 6c1fcab85602 144416f89b4a 2024-09-17T10:58:24.056 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (78s) 37s ago 14m 244M 4096M 19.3.0-4937-g74984932 6c1fcab85602 2908fd9a4510 2024-09-17T10:58:24.056 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (42s) 37s ago 14m 11.1M 4096M 19.3.0-4937-g74984932 6c1fcab85602 8f7c760b4de3 2024-09-17T10:58:24.056 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (7m) 5s ago 17m 55.7M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:58:24.489 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:58:24.489 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:58:24.490 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T10:58:24.491 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 13 2024-09-17T10:58:24.491 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:58:24.491 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:58:24.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:24.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: pgmap v307: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 8.6 MiB/s wr, 358 op/s 2024-09-17T10:58:24.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: Detected new or changed devices on smithi006 2024-09-17T10:58:24.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:24.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi149.hmuwiq"]} : dispatch 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: Upgrade: It appears safe to stop mds.foofs.smithi149.hmuwiq 2024-09-17T10:58:24.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:24 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2244383722' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.853 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:24.853 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: pgmap v307: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 8.6 MiB/s wr, 358 op/s 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: Detected new or changed devices on smithi006 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.855 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.855 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi149.hmuwiq"]} : dispatch 2024-09-17T10:58:24.855 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: Upgrade: It appears safe to stop mds.foofs.smithi149.hmuwiq 2024-09-17T10:58:24.855 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:24 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2244383722' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:24.855 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:58:24.855 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:58:24.855 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:58:24.855 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:58:24.855 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:58:24.855 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:58:24.855 INFO:teuthology.orchestra.run.smithi006.stdout: "osd", 2024-09-17T10:58:24.856 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-09-17T10:58:24.856 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-09-17T10:58:24.856 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:58:24.856 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "15/23 daemons upgraded", 2024-09-17T10:58:24.856 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mds daemons", 2024-09-17T10:58:24.856 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:58:24.856 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:58:25.287 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:58:25.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:25 smithi006 ceph-mon[63965]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:25.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:25 smithi006 ceph-mon[63965]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:25.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:25 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/3949457485' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:58:25.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:25 smithi149 ceph-mon[49697]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:25.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:25 smithi149 ceph-mon[49697]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:25.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:25 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/3949457485' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:58:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:26 smithi149 ceph-mon[49697]: pgmap v308: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.1 MiB/s wr, 250 op/s 2024-09-17T10:58:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:26 smithi149 ceph-mon[49697]: Upgrade: Updating mds.foofs.smithi149.hmuwiq 2024-09-17T10:58:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.hmuwiq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T10:58:26.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:26 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:26.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:26 smithi149 ceph-mon[49697]: Deploying daemon mds.foofs.smithi149.hmuwiq on smithi149 2024-09-17T10:58:26.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:26 smithi006 ceph-mon[63965]: pgmap v308: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.1 MiB/s wr, 250 op/s 2024-09-17T10:58:26.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:26 smithi006 ceph-mon[63965]: Upgrade: Updating mds.foofs.smithi149.hmuwiq 2024-09-17T10:58:26.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:26.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.hmuwiq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T10:58:26.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:26 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:26.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:26 smithi006 ceph-mon[63965]: Deploying daemon mds.foofs.smithi149.hmuwiq on smithi149 2024-09-17T10:58:27.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[63961]: 2024-09-17T10:58:27.550+0000 7f8897eac640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:58:27.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-mon[63965]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T10:58:27.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-mon[63965]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:58:27.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-mon[63965]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T10:58:27.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-mon[63965]: Standby daemon mds.foofs.smithi006.nripkg assigned to filesystem foofs as rank 0 2024-09-17T10:58:27.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-mon[63965]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T10:58:27.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-mon[63965]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T10:58:27.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-mon[63965]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T10:58:27.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:27 smithi006 ceph-mon[63965]: fsmap foofs:1/1 {0=foofs.smithi006.nripkg=up:replay} 2024-09-17T10:58:27.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:58:27.579+0000 7f48a3a0c640 -1 bdev(0x5641b61fec00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-17T10:58:27.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:58:27.580+0000 7f48a2a0a640 -1 bdev(0x5641b61fec00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-17T10:58:27.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T10:58:27.582+0000 7f48a2209640 -1 bdev(0x5641b61fec00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-17T10:58:27.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-mon[49697]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T10:58:27.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-mon[49697]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T10:58:27.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-mon[49697]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T10:58:27.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-mon[49697]: Standby daemon mds.foofs.smithi006.nripkg assigned to filesystem foofs as rank 0 2024-09-17T10:58:27.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-mon[49697]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T10:58:27.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-mon[49697]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T10:58:27.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-mon[49697]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T10:58:27.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:27 smithi149 ceph-mon[49697]: fsmap foofs:1/1 {0=foofs.smithi006.nripkg=up:replay} 2024-09-17T10:58:28.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:28 smithi006 ceph-mon[63965]: pgmap v310: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 328 KiB/s wr, 54 op/s 2024-09-17T10:58:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:28 smithi149 ceph-mon[49697]: pgmap v310: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 328 KiB/s wr, 54 op/s 2024-09-17T10:58:29.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:29 smithi006 ceph-mon[63965]: reconnect by client.14556 172.21.15.6:0/116617968 after 0.00099999 2024-09-17T10:58:29.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:29 smithi006 ceph-mon[63965]: reconnect by client.24347 172.21.15.149:0/3560729512 after 0.00099999 2024-09-17T10:58:29.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:29 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771] up:reconnect 2024-09-17T10:58:29.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:29 smithi006 ceph-mon[63965]: fsmap foofs:1/1 {0=foofs.smithi006.nripkg=up:reconnect} 2024-09-17T10:58:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:29 smithi149 ceph-mon[49697]: reconnect by client.14556 172.21.15.6:0/116617968 after 0.00099999 2024-09-17T10:58:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:29 smithi149 ceph-mon[49697]: reconnect by client.24347 172.21.15.149:0/3560729512 after 0.00099999 2024-09-17T10:58:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:29 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771] up:reconnect 2024-09-17T10:58:29.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:29 smithi149 ceph-mon[49697]: fsmap foofs:1/1 {0=foofs.smithi006.nripkg=up:reconnect} 2024-09-17T10:58:30.910 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:30 smithi149 ceph-mon[49697]: pgmap v311: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 63 KiB/s wr, 36 op/s 2024-09-17T10:58:30.910 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:30 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771] up:rejoin 2024-09-17T10:58:30.910 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:30 smithi149 ceph-mon[49697]: fsmap foofs:1/1 {0=foofs.smithi006.nripkg=up:rejoin} 2024-09-17T10:58:30.910 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:30 smithi149 ceph-mon[49697]: daemon mds.foofs.smithi006.nripkg is now active in filesystem foofs as rank 0 2024-09-17T10:58:30.910 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:30.910 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:58:30.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:30 smithi006 ceph-mon[63965]: pgmap v311: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 63 KiB/s wr, 36 op/s 2024-09-17T10:58:30.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:30 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771] up:rejoin 2024-09-17T10:58:30.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:30 smithi006 ceph-mon[63965]: fsmap foofs:1/1 {0=foofs.smithi006.nripkg=up:rejoin} 2024-09-17T10:58:30.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:30 smithi006 ceph-mon[63965]: daemon mds.foofs.smithi006.nripkg is now active in filesystem foofs as rank 0 2024-09-17T10:58:30.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:30.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:58:31.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:31 smithi006 ceph-mon[63965]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T10:58:31.953 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:31 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771] up:active 2024-09-17T10:58:31.953 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:31 smithi006 ceph-mon[63965]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 2024-09-17T10:58:31.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:31 smithi149 ceph-mon[49697]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T10:58:31.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:31 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771] up:active 2024-09-17T10:58:31.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:31 smithi149 ceph-mon[49697]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 2024-09-17T10:58:32.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:32 smithi006 ceph-mon[63965]: pgmap v312: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 63 KiB/s wr, 20 op/s 2024-09-17T10:58:32.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:32.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:32.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:32 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:32 smithi149 ceph-mon[49697]: pgmap v312: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 63 KiB/s wr, 20 op/s 2024-09-17T10:58:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:32.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:32 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:33.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:33 smithi006 ceph-mon[63965]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T10:58:33.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:33 smithi006 ceph-mon[63965]: Cluster is now healthy 2024-09-17T10:58:33.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:33 smithi006 ceph-mon[63965]: mds.? [v2:172.21.15.149:6832/1129497713,v1:172.21.15.149:6833/1129497713] up:boot 2024-09-17T10:58:33.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:33 smithi006 ceph-mon[63965]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:58:33.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:33 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"} : dispatch 2024-09-17T10:58:33.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:33 smithi149 ceph-mon[49697]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T10:58:33.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:33 smithi149 ceph-mon[49697]: Cluster is now healthy 2024-09-17T10:58:33.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:33 smithi149 ceph-mon[49697]: mds.? [v2:172.21.15.149:6832/1129497713,v1:172.21.15.149:6833/1129497713] up:boot 2024-09-17T10:58:33.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:33 smithi149 ceph-mon[49697]: fsmap foofs:1 {0=foofs.smithi006.nripkg=up:active} 1 up:standby 2024-09-17T10:58:33.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:33 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "mds metadata", "who": "foofs.smithi149.hmuwiq"} : dispatch 2024-09-17T10:58:34.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:34 smithi006 ceph-mon[63965]: pgmap v313: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 9.2 KiB/s wr, 11 op/s 2024-09-17T10:58:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:34 smithi149 ceph-mon[49697]: pgmap v313: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 9.2 KiB/s wr, 11 op/s 2024-09-17T10:58:36.296 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:36.296 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:36.296 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:36.296 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:36 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:36.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:36.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:36.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:36.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:36 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:37.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:37 smithi149 ceph-mon[49697]: pgmap v314: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 9.2 KiB/s wr, 11 op/s 2024-09-17T10:58:37.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:37 smithi006 ceph-mon[63965]: pgmap v314: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 9.2 KiB/s wr, 11 op/s 2024-09-17T10:58:38.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:38 smithi006 ceph-mon[63965]: pgmap v315: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-09-17T10:58:38.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:38 smithi149 ceph-mon[49697]: pgmap v315: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-09-17T10:58:39.969 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: Detected new or changed devices on smithi149 2024-09-17T10:58:39.969 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:39.969 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:39.969 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:39.969 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi006.nripkg"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi006.nripkg"}]': finished 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi149.hmuwiq"} : dispatch 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi149.hmuwiq"}]': finished 2024-09-17T10:58:39.970 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:39.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:39.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:39.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:39.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:39 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: Detected new or changed devices on smithi149 2024-09-17T10:58:40.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi006.nripkg"} : dispatch 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi006.nripkg"}]': finished 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi149.hmuwiq"} : dispatch 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi149.hmuwiq"}]': finished 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:40.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:40.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:39 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:41.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all mds 2024-09-17T10:58:41.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all rgw 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all iscsi 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: pgmap v316: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T10:58:41.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:40 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T10:58:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all mds 2024-09-17T10:58:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all rgw 2024-09-17T10:58:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T10:58:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T10:58:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all iscsi 2024-09-17T10:58:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: pgmap v316: 97 pgs: 97 active+clean; 236 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-17T10:58:41.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:41.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T10:58:41.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T10:58:41.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T10:58:41.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:41.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T10:58:41.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:40 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T10:58:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: Upgrade: Updating nfs.foo.0.0.smithi006.izithr (1/2) 2024-09-17T10:58:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: Creating key for client.nfs.foo.0.0.smithi006.izithr 2024-09-17T10:58:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T10:58:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: Rados config object exists: conf-nfs.foo 2024-09-17T10:58:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: Creating key for client.nfs.foo.0.0.smithi006.izithr-rgw 2024-09-17T10:58:42.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T10:58:42.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: Bind address in nfs.foo.0.0.smithi006.izithr's ganesha conf is defaulting to empty 2024-09-17T10:58:42.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:42.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:41 smithi006 ceph-mon[63965]: Deploying daemon nfs.foo.0.0.smithi006.izithr on smithi006 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: Upgrade: Updating nfs.foo.0.0.smithi006.izithr (1/2) 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: Creating key for client.nfs.foo.0.0.smithi006.izithr 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: Rados config object exists: conf-nfs.foo 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: Creating key for client.nfs.foo.0.0.smithi006.izithr-rgw 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.izithr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: Bind address in nfs.foo.0.0.smithi006.izithr's ganesha conf is defaulting to empty 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:42.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:41 smithi149 ceph-mon[49697]: Deploying daemon nfs.foo.0.0.smithi006.izithr on smithi006 2024-09-17T10:58:42.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:42 smithi006 ceph-mon[63965]: pgmap v317: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-17T10:58:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:42 smithi149 ceph-mon[49697]: pgmap v317: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-17T10:58:44.887 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:44 smithi006 ceph-mon[63965]: pgmap v318: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 7.3 KiB/s wr, 8 op/s 2024-09-17T10:58:44.887 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:44 smithi006 ceph-mon[63965]: from='mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771]' entity='mds.foofs.smithi006.nripkg' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.6:0/116617968"} : dispatch 2024-09-17T10:58:44.887 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:44 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:44.887 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:44 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:44 smithi149 ceph-mon[49697]: pgmap v318: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 7.3 KiB/s wr, 8 op/s 2024-09-17T10:58:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:44 smithi149 ceph-mon[49697]: from='mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771]' entity='mds.foofs.smithi006.nripkg' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.6:0/116617968"} : dispatch 2024-09-17T10:58:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:44 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:44 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:46.179 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:45 smithi149 ceph-mon[49697]: Evicting (and blocklisting) client session 14556 (172.21.15.6:0/116617968) 2024-09-17T10:58:46.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:45 smithi149 ceph-mon[49697]: from='mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771]' entity='mds.foofs.smithi006.nripkg' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.6:0/116617968"}]': finished 2024-09-17T10:58:46.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:45 smithi149 ceph-mon[49697]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T10:58:46.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:46.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:58:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:45 smithi006 ceph-mon[63965]: Evicting (and blocklisting) client session 14556 (172.21.15.6:0/116617968) 2024-09-17T10:58:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:45 smithi006 ceph-mon[63965]: from='mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771]' entity='mds.foofs.smithi006.nripkg' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.6:0/116617968"}]': finished 2024-09-17T10:58:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:45 smithi006 ceph-mon[63965]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T10:58:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:46.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:58:47.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: pgmap v320: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 8.6 KiB/s wr, 3 op/s 2024-09-17T10:58:47.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:47.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T10:58:47.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T10:58:47.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T10:58:47.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:47.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T10:58:47.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T10:58:47.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T10:58:47.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:47 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: pgmap v320: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 8.6 KiB/s wr, 3 op/s 2024-09-17T10:58:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T10:58:47.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T10:58:47.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T10:58:47.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:47.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T10:58:47.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T10:58:47.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.ppflwc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T10:58:47.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:47 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:58:48.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:48 smithi149 ceph-mon[49697]: Upgrade: Updating nfs.foo.1.0.smithi149.ppflwc (2/2) 2024-09-17T10:58:48.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:48 smithi149 ceph-mon[49697]: Creating key for client.nfs.foo.1.0.smithi149.ppflwc 2024-09-17T10:58:48.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:48 smithi149 ceph-mon[49697]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T10:58:48.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:48 smithi149 ceph-mon[49697]: Rados config object exists: conf-nfs.foo 2024-09-17T10:58:48.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:48 smithi149 ceph-mon[49697]: Creating key for client.nfs.foo.1.0.smithi149.ppflwc-rgw 2024-09-17T10:58:48.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:48 smithi149 ceph-mon[49697]: Bind address in nfs.foo.1.0.smithi149.ppflwc's ganesha conf is defaulting to empty 2024-09-17T10:58:48.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:48 smithi149 ceph-mon[49697]: Deploying daemon nfs.foo.1.0.smithi149.ppflwc on smithi149 2024-09-17T10:58:48.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:48 smithi006 ceph-mon[63965]: Upgrade: Updating nfs.foo.1.0.smithi149.ppflwc (2/2) 2024-09-17T10:58:48.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:48 smithi006 ceph-mon[63965]: Creating key for client.nfs.foo.1.0.smithi149.ppflwc 2024-09-17T10:58:48.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:48 smithi006 ceph-mon[63965]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T10:58:48.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:48 smithi006 ceph-mon[63965]: Rados config object exists: conf-nfs.foo 2024-09-17T10:58:48.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:48 smithi006 ceph-mon[63965]: Creating key for client.nfs.foo.1.0.smithi149.ppflwc-rgw 2024-09-17T10:58:48.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:48 smithi006 ceph-mon[63965]: Bind address in nfs.foo.1.0.smithi149.ppflwc's ganesha conf is defaulting to empty 2024-09-17T10:58:48.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:48 smithi006 ceph-mon[63965]: Deploying daemon nfs.foo.1.0.smithi149.ppflwc on smithi149 2024-09-17T10:58:49.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:49 smithi149 ceph-mon[49697]: pgmap v321: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-17T10:58:49.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:49 smithi006 ceph-mon[63965]: pgmap v321: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-17T10:58:51.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:50 smithi006 ceph-mon[63965]: pgmap v322: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-17T10:58:51.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:51.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:51.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:50 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:50 smithi149 ceph-mon[49697]: pgmap v322: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-17T10:58:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:51.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:50 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:58:53.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:52 smithi006 ceph-mon[63965]: pgmap v323: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-17T10:58:53.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:53.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:52 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:52 smithi149 ceph-mon[49697]: pgmap v323: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-17T10:58:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:53.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:52 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:54.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:53 smithi006 ceph-mon[63965]: from='mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771]' entity='mds.foofs.smithi006.nripkg' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.149:0/3560729512"} : dispatch 2024-09-17T10:58:54.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:53 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:54.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:53 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:54.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:53 smithi149 ceph-mon[49697]: from='mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771]' entity='mds.foofs.smithi006.nripkg' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.149:0/3560729512"} : dispatch 2024-09-17T10:58:54.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:53 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:54.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:53 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:55.233 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:54 smithi006 ceph-mon[63965]: Evicting (and blocklisting) client session 24347 (172.21.15.149:0/3560729512) 2024-09-17T10:58:55.233 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:54 smithi006 ceph-mon[63965]: pgmap v324: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-17T10:58:55.233 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:54 smithi006 ceph-mon[63965]: from='mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771]' entity='mds.foofs.smithi006.nripkg' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.149:0/3560729512"}]': finished 2024-09-17T10:58:55.233 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:54 smithi006 ceph-mon[63965]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T10:58:55.251 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:54 smithi149 ceph-mon[49697]: Evicting (and blocklisting) client session 24347 (172.21.15.149:0/3560729512) 2024-09-17T10:58:55.251 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:54 smithi149 ceph-mon[49697]: pgmap v324: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-17T10:58:55.251 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:54 smithi149 ceph-mon[49697]: from='mds.? [v2:172.21.15.6:6834/929675771,v1:172.21.15.6:6835/929675771]' entity='mds.foofs.smithi006.nripkg' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.149:0/3560729512"}]': finished 2024-09-17T10:58:55.251 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:54 smithi149 ceph-mon[49697]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T10:58:55.693 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (10m) 0s ago 19m 20.6M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (6m) 0s ago 19m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (6m) 3s ago 18m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (10m) 0s ago 19m 52.4M - dad864ee21e9 89df198e1a9b 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (45s) 0s ago 13m 29.9M - 19.3.0-4937-g74984932 6c1fcab85602 7492dcce75c5 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (24s) 3s ago 13m 13.2M - 19.3.0-4937-g74984932 6c1fcab85602 4ba61be916ff 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (9m) 0s ago 20m 524M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (8m) 3s ago 18m 458M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (8m) 0s ago 20m 63.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:58:56.443 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (7m) 3s ago 18m 47.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:58:56.444 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (12s) 0s ago 13m 66.4M - 5.9 6c1fcab85602 60782ecacbff 2024-09-17T10:58:56.444 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (7s) 3s ago 12m 17.8M - 5.9 6c1fcab85602 61716087950d 2024-09-17T10:58:56.444 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (9m) 0s ago 19m 10.6M - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:58:56.444 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (9m) 3s ago 18m 10.7M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:58:56.444 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (6m) 0s ago 17m 399M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:58:56.444 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (5m) 0s ago 17m 399M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:58:56.444 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (4m) 0s ago 16m 374M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:58:56.444 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (3m) 0s ago 16m 386M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T10:58:56.445 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (3m) 3s ago 16m 338M 4096M 19.3.0-4937-g74984932 6c1fcab85602 016956823ff1 2024-09-17T10:58:56.445 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (2m) 3s ago 15m 323M 4096M 19.3.0-4937-g74984932 6c1fcab85602 144416f89b4a 2024-09-17T10:58:56.445 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (111s) 3s ago 15m 284M 4096M 19.3.0-4937-g74984932 6c1fcab85602 2908fd9a4510 2024-09-17T10:58:56.445 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (74s) 3s ago 14m 307M 4096M 19.3.0-4937-g74984932 6c1fcab85602 8f7c760b4de3 2024-09-17T10:58:56.445 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (8m) 0s ago 18m 57.8M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:58:56.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:56.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:56.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:56 smithi006 ceph-mon[63965]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-17T10:58:56.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:56 smithi006 ceph-mon[63965]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:56.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:56.702 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:56 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:56.901 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:58:56.902 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:58:56.903 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:58:56.903 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 14 2024-09-17T10:58:56.903 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:58:56.903 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:58:56.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:56.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:56.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:56 smithi149 ceph-mon[49697]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-17T10:58:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:56 smithi149 ceph-mon[49697]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:56.957 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:56 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:57.275 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "osd", 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr", 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "nfs", 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "mds", 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "crash" 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "18/23 daemons upgraded", 2024-09-17T10:58:57.276 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading nfs daemons", 2024-09-17T10:58:57.277 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T10:58:57.277 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:58:57.717 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T10:58:57.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:57 smithi006 ceph-mon[63965]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:57.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:57 smithi006 ceph-mon[63965]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:57.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:57 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:57.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:57 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:57.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:57 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/726378482' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:57.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:57 smithi149 ceph-mon[49697]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:57.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:57 smithi149 ceph-mon[49697]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:57.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:57 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:57.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:57 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:58:57.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:57 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/726378482' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:58:58.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:58 smithi006 ceph-mon[63965]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:58.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:58 smithi006 ceph-mon[63965]: pgmap v327: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-17T10:58:58.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:58:58 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1252387064' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:58:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:58 smithi149 ceph-mon[49697]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:58:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:58 smithi149 ceph-mon[49697]: pgmap v327: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-17T10:58:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:58:58 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/1252387064' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T10:59:01.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: pgmap v328: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-17T10:59:01.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:59:01.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi006.izithr"} : dispatch 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi006.izithr"}]': finished 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi149.ppflwc"} : dispatch 2024-09-17T10:59:01.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi149.ppflwc"}]': finished 2024-09-17T10:59:01.004 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.004 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.004 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.004 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.004 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: pgmap v328: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-17T10:59:01.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:59:01.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:59:01.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.208 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:59:01.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:01.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi006.izithr"} : dispatch 2024-09-17T10:59:01.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi006.izithr"}]': finished 2024-09-17T10:59:01.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi149.ppflwc"} : dispatch 2024-09-17T10:59:01.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi149.ppflwc"}]': finished 2024-09-17T10:59:01.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:01.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:01.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:02.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:01 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all nfs 2024-09-17T10:59:02.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:01 smithi006 ceph-mon[63965]: Upgrade: Setting container_image for all nvmeof 2024-09-17T10:59:02.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:01 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all nfs 2024-09-17T10:59:02.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:01 smithi149 ceph-mon[49697]: Upgrade: Setting container_image for all nvmeof 2024-09-17T10:59:03.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:02 smithi006 ceph-mon[63965]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 511 B/s wr, 1 op/s 2024-09-17T10:59:03.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:02 smithi006 ceph-mon[63965]: Upgrade: Updating grafana.smithi006 2024-09-17T10:59:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:02 smithi149 ceph-mon[49697]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 511 B/s wr, 1 op/s 2024-09-17T10:59:03.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:02 smithi149 ceph-mon[49697]: Upgrade: Updating grafana.smithi006 2024-09-17T10:59:04.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:03 smithi006 ceph-mon[63965]: Deploying daemon grafana.smithi006 on smithi006 2024-09-17T10:59:04.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:03 smithi149 ceph-mon[49697]: Deploying daemon grafana.smithi006 on smithi006 2024-09-17T10:59:05.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:04 smithi006 ceph-mon[63965]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-17T10:59:05.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:04 smithi149 ceph-mon[49697]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-17T10:59:07.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:06 smithi006 ceph-mon[63965]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 88 B/s rd, 352 B/s wr, 0 op/s 2024-09-17T10:59:07.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:06 smithi149 ceph-mon[49697]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 88 B/s rd, 352 B/s wr, 0 op/s 2024-09-17T10:59:09.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:08 smithi006 ceph-mon[63965]: pgmap v332: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T10:59:09.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:08 smithi149 ceph-mon[49697]: pgmap v332: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T10:59:10.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:10 smithi006 ceph-mon[63965]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:11.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:10 smithi149 ceph-mon[49697]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:13.063 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:12 smithi006 ceph-mon[63965]: pgmap v334: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:13.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:12 smithi149 ceph-mon[49697]: pgmap v334: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:15.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:14 smithi006 ceph-mon[63965]: pgmap v335: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:15.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:14 smithi149 ceph-mon[49697]: pgmap v335: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:15.998 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:59:15.998 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:15.998 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:15.998 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:16.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:59:16.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:16.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:16.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:17.190 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:16 smithi006 ceph-mon[63965]: pgmap v336: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:16 smithi149 ceph-mon[49697]: pgmap v336: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:18.613 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:18 smithi006 ceph-mon[63965]: pgmap v337: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:18.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:18 smithi149 ceph-mon[49697]: pgmap v337: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:20.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:20 smithi006 ceph-mon[63965]: pgmap v338: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:20 smithi149 ceph-mon[49697]: pgmap v338: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:22.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:22.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:22.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:22 smithi006 ceph-mon[63965]: pgmap v339: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:22.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:22.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:22 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:22 smithi149 ceph-mon[49697]: pgmap v339: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:22 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:24.910 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:24 smithi006 ceph-mon[63965]: pgmap v340: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:24.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:24 smithi149 ceph-mon[49697]: pgmap v340: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:26.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:26 smithi006 ceph-mon[63965]: pgmap v341: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:26.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:26 smithi149 ceph-mon[49697]: pgmap v341: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: pgmap v342: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:28.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: Upgrade: Finalizing container_image settings 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-17T10:59:28.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T10:59:28.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: Upgrade: Complete! 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:59:28.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:59:28.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-17T10:59:28.458 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T10:59:28.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: pgmap v342: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:28.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:28.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:28.460 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: Upgrade: Finalizing container_image settings 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T10:59:28.461 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: Upgrade: Complete! 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.463 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:28.773 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph orch ps' 2024-09-17T10:59:29.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:29 smithi006 ceph-mon[63965]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:59:29.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:29.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:59:29.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:59:29.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:29.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:29.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:29 smithi149 ceph-mon[49697]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:59:29.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T10:59:29.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T10:59:29.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T10:59:29.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:29.457 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:30.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:30 smithi006 ceph-mon[63965]: pgmap v343: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:30.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:59:30.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:30 smithi149 ceph-mon[49697]: pgmap v343: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:59:30.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T10:59:30.982 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (10m) 9s ago 19m 20.6M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (7m) 9s ago 19m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (7m) 38s ago 19m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (16s) 9s ago 19m 41.6M - 10.4.0 c8b91775d855 e6a34d000662 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (80s) 9s ago 13m 30.2M - 19.3.0-4937-g74984932 6c1fcab85602 7492dcce75c5 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (59s) 38s ago 13m 13.2M - 19.3.0-4937-g74984932 6c1fcab85602 4ba61be916ff 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (9m) 9s ago 21m 533M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (8m) 38s ago 19m 458M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (8m) 9s ago 21m 64.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (8m) 38s ago 19m 47.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T10:59:30.983 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (46s) 9s ago 13m 67.2M - 5.9 6c1fcab85602 60782ecacbff 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (41s) 38s ago 13m 17.8M - 5.9 6c1fcab85602 61716087950d 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (10m) 9s ago 19m 10.0M - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (10m) 38s ago 19m 10.7M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (6m) 9s ago 18m 398M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (5m) 9s ago 17m 400M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (5m) 9s ago 17m 375M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (4m) 9s ago 17m 386M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (3m) 38s ago 16m 338M 4096M 19.3.0-4937-g74984932 6c1fcab85602 016956823ff1 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (3m) 38s ago 16m 323M 4096M 19.3.0-4937-g74984932 6c1fcab85602 144416f89b4a 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (2m) 38s ago 15m 284M 4096M 19.3.0-4937-g74984932 6c1fcab85602 2908fd9a4510 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (109s) 38s ago 15m 307M 4096M 19.3.0-4937-g74984932 6c1fcab85602 8f7c760b4de3 2024-09-17T10:59:30.984 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (9m) 9s ago 18m 57.8M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T10:59:31.325 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:31 smithi006 ceph-mon[63965]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:59:31.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:31 smithi149 ceph-mon[49697]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T10:59:31.724 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph versions' 2024-09-17T10:59:32.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:32 smithi006 ceph-mon[63965]: pgmap v344: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:32.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:32 smithi149 ceph-mon[49697]: pgmap v344: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:33.918 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T10:59:33.918 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T10:59:33.918 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:59:33.918 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:59:33.918 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T10:59:33.918 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 14 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T10:59:33.919 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T10:59:34.654 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-17T10:59:34.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:34 smithi006 ceph-mon[63965]: pgmap v345: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:34.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:34 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/4128053490' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:34 smithi149 ceph-mon[49697]: pgmap v345: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:34.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:34 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/4128053490' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T10:59:36.575 INFO:teuthology.orchestra.run.smithi006.stdout:wait for servicemap items w/ changing names to refresh 2024-09-17T10:59:36.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:36 smithi006 ceph-mon[63965]: pgmap v346: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:36.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:36 smithi149 ceph-mon[49697]: pgmap v346: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:37.261 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'sleep 60' 2024-09-17T10:59:38.904 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:38 smithi006 ceph-mon[63965]: pgmap v347: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:38.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:38 smithi149 ceph-mon[49697]: pgmap v347: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:40.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:40 smithi006 ceph-mon[63965]: pgmap v348: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:40.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:40 smithi149 ceph-mon[49697]: pgmap v348: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:42.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:42 smithi006 ceph-mon[63965]: pgmap v349: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:42.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:42 smithi149 ceph-mon[49697]: pgmap v349: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:44.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:44 smithi006 ceph-mon[63965]: pgmap v350: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:44.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:44 smithi149 ceph-mon[49697]: pgmap v350: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:45.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:59:45.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T10:59:46.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:46 smithi006 ceph-mon[63965]: pgmap v351: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:46.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:46 smithi149 ceph-mon[49697]: pgmap v351: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:48.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:48 smithi006 ceph-mon[63965]: pgmap v352: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:48.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:48 smithi149 ceph-mon[49697]: pgmap v352: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:50.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:50 smithi006 ceph-mon[63965]: pgmap v353: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:50.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:50 smithi149 ceph-mon[49697]: pgmap v353: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:52.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:52 smithi006 ceph-mon[63965]: pgmap v354: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:52.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:52 smithi149 ceph-mon[49697]: pgmap v354: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:54.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:54 smithi006 ceph-mon[63965]: pgmap v355: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:54.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:54 smithi149 ceph-mon[49697]: pgmap v355: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:56.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:56 smithi006 ceph-mon[63965]: pgmap v356: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:56.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:56 smithi149 ceph-mon[49697]: pgmap v356: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T10:59:58.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 10:59:58 smithi006 ceph-mon[63965]: pgmap v357: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T10:59:58.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 10:59:58 smithi149 ceph-mon[49697]: pgmap v357: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:00.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:00 smithi006 ceph-mon[63965]: pgmap v358: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:00.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:00 smithi006 ceph-mon[63965]: overall HEALTH_OK 2024-09-17T11:00:00.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:00:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:00 smithi149 ceph-mon[49697]: pgmap v358: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:00 smithi149 ceph-mon[49697]: overall HEALTH_OK 2024-09-17T11:00:00.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:00:02.952 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:02 smithi006 ceph-mon[63965]: pgmap v359: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:02.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:02 smithi149 ceph-mon[49697]: pgmap v359: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:04.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:04 smithi006 ceph-mon[63965]: pgmap v360: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:04.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:04 smithi149 ceph-mon[49697]: pgmap v360: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:06.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:06 smithi006 ceph-mon[63965]: pgmap v361: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:06.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:06 smithi149 ceph-mon[49697]: pgmap v361: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:08.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:08 smithi006 ceph-mon[63965]: pgmap v362: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:08.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:08 smithi149 ceph-mon[49697]: pgmap v362: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:10.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:10 smithi006 ceph-mon[63965]: pgmap v363: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:10.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:10 smithi149 ceph-mon[49697]: pgmap v363: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:12.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:12 smithi149 ceph-mon[49697]: pgmap v364: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:13.112 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:12 smithi006 ceph-mon[63965]: pgmap v364: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:14.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:14 smithi149 ceph-mon[49697]: pgmap v365: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:15.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:14 smithi006 ceph-mon[63965]: pgmap v365: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:16.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:15 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:00:16.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:15 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:00:17.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:16 smithi006 ceph-mon[63965]: pgmap v366: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:17.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:16 smithi149 ceph-mon[49697]: pgmap v366: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:18.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:18 smithi006 ceph-mon[63965]: pgmap v367: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:18.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:18 smithi149 ceph-mon[49697]: pgmap v367: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:20.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:20 smithi006 ceph-mon[63965]: pgmap v368: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:20.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:20 smithi149 ceph-mon[49697]: pgmap v368: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:22.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:22 smithi006 ceph-mon[63965]: pgmap v369: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:22.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:22 smithi149 ceph-mon[49697]: pgmap v369: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:24.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:24 smithi006 ceph-mon[63965]: pgmap v370: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:24.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:24 smithi149 ceph-mon[49697]: pgmap v370: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:26.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:26 smithi006 ceph-mon[63965]: pgmap v371: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:26.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:26 smithi149 ceph-mon[49697]: pgmap v371: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:28.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:28 smithi006 ceph-mon[63965]: pgmap v372: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:28.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:28 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T11:00:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:28 smithi149 ceph-mon[49697]: pgmap v372: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:28.956 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:28 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T11:00:30.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T11:00:30.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T11:00:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T11:00:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T11:00:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T11:00:30.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:29 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T11:00:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T11:00:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T11:00:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T11:00:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T11:00:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T11:00:30.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:29 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' 2024-09-17T11:00:31.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:30 smithi006 ceph-mon[63965]: pgmap v373: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:31.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:30 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:00:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:30 smithi149 ceph-mon[49697]: pgmap v373: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:31.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:30 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:00:33.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:32 smithi006 ceph-mon[63965]: pgmap v374: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:33.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:32 smithi149 ceph-mon[49697]: pgmap v374: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:35.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:34 smithi006 ceph-mon[63965]: pgmap v375: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:35.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:34 smithi149 ceph-mon[49697]: pgmap v375: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:37.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:36 smithi006 ceph-mon[63965]: pgmap v376: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:37.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:36 smithi149 ceph-mon[49697]: pgmap v376: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:39.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:38 smithi006 ceph-mon[63965]: pgmap v377: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:39.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:38 smithi149 ceph-mon[49697]: pgmap v377: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:39.956 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph orch ps' 2024-09-17T11:00:41.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:40 smithi006 ceph-mon[63965]: pgmap v378: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:41.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:40 smithi149 ceph-mon[49697]: pgmap v378: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:42.129 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (12m) 80s ago 21m 20.6M - 0.27.0 11f11916f8cd c9859da4d4d9 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (8m) 80s ago 21m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 ef7a6e4be48b 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi149 smithi149 running (8m) 109s ago 20m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 1c104cb1dd10 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (87s) 80s ago 20m 41.6M - 10.4.0 c8b91775d855 e6a34d000662 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.nripkg smithi006 running (2m) 80s ago 15m 30.2M - 19.3.0-4937-g74984932 6c1fcab85602 7492dcce75c5 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi149.hmuwiq smithi149 running (2m) 109s ago 15m 13.2M - 19.3.0-4937-g74984932 6c1fcab85602 4ba61be916ff 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.plhvdm smithi006 *:8443,9283,8765 running (11m) 80s ago 22m 533M - 19.3.0-4937-g74984932 6c1fcab85602 01382dbfa2e8 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi149.ijckag smithi149 *:8443,9283,8765 running (10m) 109s ago 20m 458M - 19.3.0-4937-g74984932 6c1fcab85602 62d9904f6a83 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (9m) 80s ago 22m 64.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 241e5f24b4c7 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi149 smithi149 running (9m) 109s ago 20m 47.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 800698b81fca 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.izithr smithi006 *:2049 running (117s) 80s ago 14m 67.2M - 5.9 6c1fcab85602 60782ecacbff 2024-09-17T11:00:42.130 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi149.ppflwc smithi149 *:2049 running (112s) 109s ago 14m 17.8M - 5.9 6c1fcab85602 61716087950d 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (11m) 80s ago 20m 10.0M - 1.7.0 72c9c2088986 e97346498d31 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi149 smithi149 *:9100 running (11m) 109s ago 20m 10.7M - 1.7.0 72c9c2088986 e37d161c567c 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (7m) 80s ago 19m 398M 1937M 19.3.0-4937-g74984932 6c1fcab85602 2775f74b7e4d 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (7m) 80s ago 19m 400M 1937M 19.3.0-4937-g74984932 6c1fcab85602 35586f41af8f 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (6m) 80s ago 18m 375M 1937M 19.3.0-4937-g74984932 6c1fcab85602 8f61d308918e 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (5m) 80s ago 18m 386M 1937M 19.3.0-4937-g74984932 6c1fcab85602 654bf1272724 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi149 running (4m) 109s ago 17m 338M 4096M 19.3.0-4937-g74984932 6c1fcab85602 016956823ff1 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi149 running (4m) 109s ago 17m 323M 4096M 19.3.0-4937-g74984932 6c1fcab85602 144416f89b4a 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi149 running (3m) 109s ago 17m 284M 4096M 19.3.0-4937-g74984932 6c1fcab85602 2908fd9a4510 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi149 running (3m) 109s ago 16m 307M 4096M 19.3.0-4937-g74984932 6c1fcab85602 8f7c760b4de3 2024-09-17T11:00:42.131 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (10m) 80s ago 20m 57.8M - 2.51.0 1d3b7f56885b 5b486a43fe0c 2024-09-17T11:00:42.760 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph orch upgrade status' 2024-09-17T11:00:43.071 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:42 smithi006 ceph-mon[63965]: pgmap v379: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:43.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:42 smithi149 ceph-mon[49697]: pgmap v379: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:44.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:43 smithi006 ceph-mon[63965]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:00:44.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:43 smithi149 ceph-mon[49697]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:00:44.936 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T11:00:44.937 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": null, 2024-09-17T11:00:44.937 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": false, 2024-09-17T11:00:44.937 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "", 2024-09-17T11:00:44.937 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-09-17T11:00:44.937 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": null, 2024-09-17T11:00:44.937 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-09-17T11:00:44.937 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-09-17T11:00:44.937 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T11:00:45.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:44 smithi006 ceph-mon[63965]: pgmap v380: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:45.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:44 smithi149 ceph-mon[49697]: pgmap v380: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:45.575 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph health detail' 2024-09-17T11:00:46.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:45 smithi006 ceph-mon[63965]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:00:46.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:45 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:00:46.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:45 smithi149 ceph-mon[49697]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:00:46.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:45 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:00:47.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:46 smithi006 ceph-mon[63965]: pgmap v381: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:47.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:46 smithi149 ceph-mon[49697]: pgmap v381: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:47.705 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-09-17T11:00:48.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:47 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2113552015' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T11:00:48.313 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph versions' 2024-09-17T11:00:48.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:47 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2113552015' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T11:00:49.322 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:48 smithi006 ceph-mon[63965]: pgmap v382: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:49.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:48 smithi149 ceph-mon[49697]: pgmap v382: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:50.479 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-09-17T11:00:50.479 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 14 2024-09-17T11:00:50.480 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T11:00:50.481 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-09-17T11:00:51.137 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:50 smithi006 ceph-mon[63965]: pgmap v383: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:51.137 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:50 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/4131242069' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T11:00:51.164 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-17T11:00:51.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:50 smithi149 ceph-mon[49697]: pgmap v383: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:51.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:50 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/4131242069' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T11:00:53.275 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-09-17T11:00:53.275 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:52 smithi006 ceph-mon[63965]: pgmap v384: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:53.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:52 smithi149 ceph-mon[49697]: pgmap v384: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:53.894 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-17T11:00:54.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:53 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/1582003530' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T11:00:54.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:53 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/1582003530' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T11:00:55.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:54 smithi006 ceph-mon[63965]: pgmap v385: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:55.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:54 smithi149 ceph-mon[49697]: pgmap v385: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:56.030 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)" 2024-09-17T11:00:56.673 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-17T11:00:56.674 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-17T11:00:56.675 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T11:00:57.258 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:56 smithi149 ceph-mon[49697]: pgmap v386: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:57.258 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:56 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2039425276' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T11:00:57.304 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:56 smithi006 ceph-mon[63965]: pgmap v386: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:00:57.304 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:56 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2039425276' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T11:00:58.737 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-09-17T11:00:58.737 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T10:39:01.555849Z", "last_refresh": "2024-09-17T10:59:21.528916Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T10:38:55.442162Z", "last_refresh": "2024-09-17T10:58:52.600941Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-17T10:38:58.348685Z", "last_refresh": "2024-09-17T10:59:21.529068Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T10:45:30.981172Z", "last_refresh": "2024-09-17T10:58:52.601539Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T10:38:54.230645Z", "last_refresh": "2024-09-17T10:58:52.601043Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T10:39:46.116436Z", "last_refresh": "2024-09-17T10:58:52.601118Z", "running": 2, "size": 2}}, {"events": ["2024-09-17T11:00:29.291965Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-17T10:45:51.775452Z", "last_refresh": "2024-09-17T10:58:52.601615Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T10:38:59.564645Z", "last_refresh": "2024-09-17T10:58:52.601190Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", "last_refresh": "2024-09-17T10:58:52.601261Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T10:38:56.617323Z", "last_refresh": "2024-09-17T10:59:21.529205Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T11:00:59.120 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:58 smithi006 ceph-mon[63965]: pgmap v387: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:00:59.342 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-17T11:00:59.343 DEBUG:teuthology.parallel:result is None 2024-09-17T11:00:59.343 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T11:00:59.356 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T11:00:59.356 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-17T11:00:59.385 INFO:teuthology.orchestra.run.smithi006.stderr:+ umount /mnt/foo 2024-09-17T11:00:59.409 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T11:00:59.419 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T11:00:59.419 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-17T11:00:59.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:58 smithi149 ceph-mon[49697]: pgmap v387: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T11:01:00.062 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:00:59 smithi006 ceph-mon[63965]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:01:00.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:00:59 smithi149 ceph-mon[49697]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:01:01.234 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:00 smithi006 ceph-mon[63965]: pgmap v388: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:01:01.234 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:00 smithi006 ceph-mon[63965]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:01:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:00 smithi149 ceph-mon[49697]: pgmap v388: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:01:01.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:00 smithi149 ceph-mon[49697]: from='mgr.34104 172.21.15.6:0/668559000' entity='mgr.smithi006.plhvdm' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T11:01:01.523 INFO:teuthology.orchestra.run.smithi006.stdout: "foo" 2024-09-17T11:01:02.184 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-17T11:01:03.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:02 smithi006 ceph-mon[63965]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:01:03.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:02 smithi006 ceph-mon[63965]: pgmap v389: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:01:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:02 smithi149 ceph-mon[49697]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:01:03.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:02 smithi149 ceph-mon[49697]: pgmap v389: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout:[ 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: { 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "access_type": "RW", 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "clients": [], 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "cluster_id": "foo", 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "export_id": 1, 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "fsal": { 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "fs_name": "foofs", 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "name": "CEPH", 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "user_id": "nfs.foo.1" 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-09-17T11:01:04.217 INFO:teuthology.orchestra.run.smithi006.stdout: "path": "/", 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: "protocols": [ 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: 4 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: "pseudo": "/fake", 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: "security_label": true, 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: "squash": "none", 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: "transports": [ 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: "TCP" 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: ] 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-09-17T11:01:04.218 INFO:teuthology.orchestra.run.smithi006.stdout:] 2024-09-17T11:01:04.906 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-17T11:01:05.201 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:04 smithi006 ceph-mon[63965]: pgmap v390: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:01:05.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:04 smithi149 ceph-mon[49697]: pgmap v390: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:01:06.363 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:05 smithi006 ceph-mon[63965]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:01:06.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:05 smithi149 ceph-mon[49697]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:01:06.859 INFO:teuthology.orchestra.run.smithi006.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-17T11:01:06.859 INFO:teuthology.orchestra.run.smithi006.stdout:foo grace 2024-09-17T11:01:06.859 INFO:teuthology.orchestra.run.smithi006.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-17T11:01:06.860 INFO:teuthology.orchestra.run.smithi006.stdout:foo conf-nfs.foo 2024-09-17T11:01:06.860 INFO:teuthology.orchestra.run.smithi006.stdout:foo export-1 2024-09-17T11:01:07.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:06 smithi006 ceph-mon[63965]: pgmap v391: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:01:07.422 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:06 smithi006 ceph-mon[63965]: mgrmap e44: smithi006.plhvdm(active, since 10m), standbys: smithi149.ijckag 2024-09-17T11:01:07.448 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T11:01:07.458 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T11:01:07.458 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ddf74116-74e0-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-17T11:01:07.458 DEBUG:teuthology.orchestra.run.smithi006:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-17T11:01:07.458 DEBUG:teuthology.orchestra.run.smithi006:> ' 2024-09-17T11:01:07.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:06 smithi149 ceph-mon[49697]: pgmap v391: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T11:01:07.459 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:06 smithi149 ceph-mon[49697]: mgrmap e44: smithi006.plhvdm(active, since 10m), standbys: smithi149.ijckag 2024-09-17T11:01:09.140 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:08 smithi006 ceph-mon[63965]: pgmap v392: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T11:01:09.223 INFO:teuthology.orchestra.run.smithi006.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-17T11:01:09.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:08 smithi149 ceph-mon[49697]: pgmap v392: 97 pgs: 97 active+clean; 236 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T11:01:09.500 INFO:teuthology.orchestra.run.smithi006.stderr:+ [[ 8 -gt 2 ]] 2024-09-17T11:01:10.126 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:10 smithi006 ceph-mon[63965]: from='client.? 172.21.15.6:0/2883808742' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-17T11:01:10.127 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-17T11:01:10.141 INFO:tasks.cephadm:Teardown begin 2024-09-17T11:01:10.142 DEBUG:teuthology.orchestra.run.smithi006:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T11:01:10.171 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T11:01:10.206 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-17T11:01:10.206 DEBUG:teuthology.orchestra.run.smithi006:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T11:01:10.227 DEBUG:teuthology.orchestra.run.smithi149:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T11:01:10.274 INFO:tasks.cephadm:Stopping all daemons... 2024-09-17T11:01:10.274 INFO:tasks.cephadm.mon.smithi006:Stopping mon.smithi006... 2024-09-17T11:01:10.274 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006 2024-09-17T11:01:10.451 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:10 smithi006 systemd[1]: Stopping Ceph mon.smithi006 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:01:10.456 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:10 smithi149 ceph-mon[49697]: from='client.? 172.21.15.6:0/2883808742' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-17T11:01:10.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:10 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[63961]: 2024-09-17T11:01:10.605+0000 7f889d6b7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi006 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:01:10.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:10 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006[63961]: 2024-09-17T11:01:10.605+0000 7f889d6b7640 -1 mon.smithi006@0(leader) e3 *** Got Signal Terminated *** 2024-09-17T11:01:10.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:10 smithi006 podman[93072]: 2024-09-17 11:01:10.769654759 +0000 UTC m=+0.302754500 container died 241e5f24b4c7e5602cf542a79e735ae6905975938c64bf0e77276a331f169c1d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-17T11:01:11.425 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:11 smithi006 podman[93072]: 2024-09-17 11:01:11.148140112 +0000 UTC m=+0.681239851 container cleanup 241e5f24b4c7e5602cf542a79e735ae6905975938c64bf0e77276a331f169c1d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-17T11:01:11.425 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:Sep 17 11:01:11 smithi006 bash[93072]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi006 2024-09-17T11:01:11.565 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi006.service' 2024-09-17T11:01:11.621 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:01:11.621 INFO:tasks.cephadm.mon.smithi006:Stopped mon.smithi006 2024-09-17T11:01:11.621 INFO:tasks.cephadm.mon.smithi149:Stopping mon.smithi149... 2024-09-17T11:01:11.621 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi149 2024-09-17T11:01:11.909 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:11 smithi149 systemd[1]: Stopping Ceph mon.smithi149 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:01:12.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:11 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149[49693]: 2024-09-17T11:01:11.907+0000 7f5e09f23640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi149 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:01:12.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:11 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149[49693]: 2024-09-17T11:01:11.907+0000 7f5e09f23640 -1 mon.smithi149@1(peon) e3 *** Got Signal Terminated *** 2024-09-17T11:01:12.207 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:11 smithi149 podman[68542]: 2024-09-17 11:01:11.979419511 +0000 UTC m=+0.164905529 container died 800698b81fca4d586e7cd9100db5ebe3043010a02b963da8fa06e50ccb5c88e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-7498493, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-17T11:01:12.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:12 smithi149 podman[68542]: 2024-09-17 11:01:12.474229259 +0000 UTC m=+0.659715280 container cleanup 800698b81fca4d586e7cd9100db5ebe3043010a02b963da8fa06e50ccb5c88e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T11:01:12.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Sep 17 11:01:12 smithi149 bash[68542]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-mon-smithi149 2024-09-17T11:01:12.852 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@mon.smithi149.service' 2024-09-17T11:01:12.904 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:01:12.905 INFO:tasks.cephadm.mon.smithi149:Stopped mon.smithi149 2024-09-17T11:01:12.905 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-17T11:01:12.905 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.0 2024-09-17T11:01:13.201 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:12 smithi006 systemd[1]: Stopping Ceph osd.0 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:01:13.701 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:13 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[68732]: 2024-09-17T11:01:13.201+0000 7fb74655c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:01:13.701 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:13 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[68732]: 2024-09-17T11:01:13.201+0000 7fb74655c640 -1 osd.0 157 *** Got signal Terminated *** 2024-09-17T11:01:13.701 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:13 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0[68732]: 2024-09-17T11:01:13.201+0000 7fb74655c640 -1 osd.0 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T11:01:18.951 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:18 smithi006 podman[93222]: 2024-09-17 11:01:18.694579179 +0000 UTC m=+5.589838305 container died 2775f74b7e4dc7970ec76caaf2d2d1ef7e3beb5953a944f72803ebe1676b9b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-7498493, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T11:01:19.451 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:19 smithi006 podman[93222]: 2024-09-17 11:01:19.011407676 +0000 UTC m=+5.906666791 container cleanup 2775f74b7e4dc7970ec76caaf2d2d1ef7e3beb5953a944f72803ebe1676b9b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T11:01:19.451 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:19 smithi006 bash[93222]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0 2024-09-17T11:01:19.451 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:19 smithi006 podman[93238]: 2024-09-17 11:01:19.229456171 +0000 UTC m=+0.524826188 container remove 2775f74b7e4dc7970ec76caaf2d2d1ef7e3beb5953a944f72803ebe1676b9b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T11:01:19.901 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:19 smithi006 podman[93328]: 2024-09-17 11:01:19.552281077 +0000 UTC m=+0.017363552 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T11:01:19.901 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:19 smithi006 podman[93328]: 2024-09-17 11:01:19.680662835 +0000 UTC m=+0.145745301 container create 2e01f3d79e72dfbdb73f68fa0c24fd7f228f08fdf5c1f151466b4bc5715fbd6f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-17T11:01:19.901 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:19 smithi006 podman[93328]: 2024-09-17 11:01:19.899794471 +0000 UTC m=+0.364876942 container init 2e01f3d79e72dfbdb73f68fa0c24fd7f228f08fdf5c1f151466b4bc5715fbd6f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-17T11:01:20.201 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:19 smithi006 podman[93328]: 2024-09-17 11:01:19.903390766 +0000 UTC m=+0.368473234 container start 2e01f3d79e72dfbdb73f68fa0c24fd7f228f08fdf5c1f151466b4bc5715fbd6f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-7498493, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T11:01:20.201 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:19 smithi006 podman[93328]: 2024-09-17 11:01:19.99149025 +0000 UTC m=+0.456572728 container attach 2e01f3d79e72dfbdb73f68fa0c24fd7f228f08fdf5c1f151466b4bc5715fbd6f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-17T11:01:20.201 INFO:journalctl@ceph.osd.0.smithi006.stdout:Sep 17 11:01:20 smithi006 podman[93328]: 2024-09-17 11:01:20.098549274 +0000 UTC m=+0.563631740 container died 2e01f3d79e72dfbdb73f68fa0c24fd7f228f08fdf5c1f151466b4bc5715fbd6f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-7498493) 2024-09-17T11:01:20.763 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.0.service' 2024-09-17T11:01:20.808 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:01:20.809 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-17T11:01:20.809 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-17T11:01:20.809 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.1 2024-09-17T11:01:21.100 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:20 smithi006 systemd[1]: Stopping Ceph osd.1 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:01:21.451 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:21 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[71176]: 2024-09-17T11:01:21.098+0000 7eff2a13e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:01:21.451 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:21 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[71176]: 2024-09-17T11:01:21.098+0000 7eff2a13e640 -1 osd.1 157 *** Got signal Terminated *** 2024-09-17T11:01:21.451 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:21 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1[71176]: 2024-09-17T11:01:21.098+0000 7eff2a13e640 -1 osd.1 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T11:01:26.858 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:26 smithi006 podman[93431]: 2024-09-17 11:01:26.527646934 +0000 UTC m=+5.521098140 container died 35586f41af8fcaaf0b2a06ae61022d3f279ce06e2ecb9a8963d9926060c02814 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T11:01:26.858 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:26 smithi006 podman[93431]: 2024-09-17 11:01:26.857258843 +0000 UTC m=+5.850710047 container cleanup 35586f41af8fcaaf0b2a06ae61022d3f279ce06e2ecb9a8963d9926060c02814 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T11:01:27.151 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:26 smithi006 bash[93431]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1 2024-09-17T11:01:27.451 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:27 smithi006 podman[93449]: 2024-09-17 11:01:27.150409964 +0000 UTC m=+0.615992663 container remove 35586f41af8fcaaf0b2a06ae61022d3f279ce06e2ecb9a8963d9926060c02814 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T11:01:27.813 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:27 smithi006 podman[93538]: 2024-09-17 11:01:27.385330641 +0000 UTC m=+0.016752576 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T11:01:27.813 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:27 smithi006 podman[93538]: 2024-09-17 11:01:27.501697718 +0000 UTC m=+0.133119633 container create 79c51776277d2c94bd01941b08d0bc98b4138259806b34bc5a7bc0a9b89a15b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T11:01:27.813 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:27 smithi006 podman[93538]: 2024-09-17 11:01:27.720822383 +0000 UTC m=+0.352244310 container init 79c51776277d2c94bd01941b08d0bc98b4138259806b34bc5a7bc0a9b89a15b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=main-7498493) 2024-09-17T11:01:27.814 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:27 smithi006 podman[93538]: 2024-09-17 11:01:27.724296871 +0000 UTC m=+0.355718787 container start 79c51776277d2c94bd01941b08d0bc98b4138259806b34bc5a7bc0a9b89a15b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-7498493, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T11:01:27.814 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:27 smithi006 podman[93538]: 2024-09-17 11:01:27.812577631 +0000 UTC m=+0.443999567 container attach 79c51776277d2c94bd01941b08d0bc98b4138259806b34bc5a7bc0a9b89a15b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T11:01:28.192 INFO:journalctl@ceph.osd.1.smithi006.stdout:Sep 17 11:01:27 smithi006 podman[93538]: 2024-09-17 11:01:27.918708166 +0000 UTC m=+0.550130093 container died 79c51776277d2c94bd01941b08d0bc98b4138259806b34bc5a7bc0a9b89a15b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-7498493) 2024-09-17T11:01:28.501 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.1.service' 2024-09-17T11:01:28.543 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:01:28.544 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-17T11:01:28.544 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-17T11:01:28.544 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.2 2024-09-17T11:01:28.879 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:28 smithi006 systemd[1]: Stopping Ceph osd.2 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:01:29.201 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:28 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[73506]: 2024-09-17T11:01:28.877+0000 7f0e03217640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:01:29.201 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:28 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[73506]: 2024-09-17T11:01:28.877+0000 7f0e03217640 -1 osd.2 157 *** Got signal Terminated *** 2024-09-17T11:01:29.201 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:28 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2[73506]: 2024-09-17T11:01:28.877+0000 7f0e03217640 -1 osd.2 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T11:01:34.697 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:34 smithi006 podman[93641]: 2024-09-17 11:01:34.35664908 +0000 UTC m=+5.583960209 container died 8f61d308918e61591979e60eb830c97c2ddfddcb69fe4154e4bb4c6ec1c5c843 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-17T11:01:34.951 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:34 smithi006 podman[93641]: 2024-09-17 11:01:34.769709962 +0000 UTC m=+5.997021090 container cleanup 8f61d308918e61591979e60eb830c97c2ddfddcb69fe4154e4bb4c6ec1c5c843 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T11:01:34.951 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:34 smithi006 bash[93641]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2 2024-09-17T11:01:35.327 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:35 smithi006 podman[93675]: 2024-09-17 11:01:35.004372492 +0000 UTC m=+0.635570826 container remove 8f61d308918e61591979e60eb830c97c2ddfddcb69fe4154e4bb4c6ec1c5c843 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-7498493, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T11:01:35.592 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:35 smithi006 podman[93761]: 2024-09-17 11:01:35.229715376 +0000 UTC m=+0.016476697 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T11:01:35.592 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:35 smithi006 podman[93761]: 2024-09-17 11:01:35.346944537 +0000 UTC m=+0.133705857 container create 70b41ba29b53c7ddc5bdc11fff92184389c774cadf7aaf8dd8e878c05c0003ff (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-17T11:01:35.592 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:35 smithi006 podman[93761]: 2024-09-17 11:01:35.591062394 +0000 UTC m=+0.377823715 container init 70b41ba29b53c7ddc5bdc11fff92184389c774cadf7aaf8dd8e878c05c0003ff (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T11:01:35.951 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:35 smithi006 podman[93761]: 2024-09-17 11:01:35.595894004 +0000 UTC m=+0.382655325 container start 70b41ba29b53c7ddc5bdc11fff92184389c774cadf7aaf8dd8e878c05c0003ff (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-7498493, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-17T11:01:35.951 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:35 smithi006 podman[93761]: 2024-09-17 11:01:35.682867737 +0000 UTC m=+0.469629067 container attach 70b41ba29b53c7ddc5bdc11fff92184389c774cadf7aaf8dd8e878c05c0003ff (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True) 2024-09-17T11:01:35.951 INFO:journalctl@ceph.osd.2.smithi006.stdout:Sep 17 11:01:35 smithi006 podman[93761]: 2024-09-17 11:01:35.802910725 +0000 UTC m=+0.589672045 container died 70b41ba29b53c7ddc5bdc11fff92184389c774cadf7aaf8dd8e878c05c0003ff (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T11:01:36.435 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.2.service' 2024-09-17T11:01:36.478 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:01:36.478 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-17T11:01:36.478 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-17T11:01:36.478 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.3 2024-09-17T11:01:36.701 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:36 smithi006 systemd[1]: Stopping Ceph osd.3 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:01:37.201 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:36 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[76002]: 2024-09-17T11:01:36.823+0000 7ff55c144640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:01:37.201 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:36 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[76002]: 2024-09-17T11:01:36.823+0000 7ff55c144640 -1 osd.3 157 *** Got signal Terminated *** 2024-09-17T11:01:37.201 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:36 smithi006 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3[76002]: 2024-09-17T11:01:36.823+0000 7ff55c144640 -1 osd.3 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T11:01:39.456 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:39 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:39.089+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:39.456 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:39 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:39.188+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:40.456 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:40 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:40.059+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:40.456 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:40 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:40.231+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:41.456 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:41.273+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:41.457 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:41.106+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:41.457 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:41.084+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:41.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:41 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:41.459+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:42.427 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:42 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:42.136+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:42.427 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:42 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:42.246+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:42.427 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:42 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:42.058+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:42.701 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:42 smithi006 podman[93865]: 2024-09-17 11:01:42.426750175 +0000 UTC m=+5.716023514 container died 654bf1272724d1afe4d81d46fbfb4ae9d2319ac96004b37cc0283d06a2cd3b35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T11:01:42.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:42 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:42.425+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:43.033 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:42 smithi006 podman[93865]: 2024-09-17 11:01:42.79763506 +0000 UTC m=+6.086908390 container cleanup 654bf1272724d1afe4d81d46fbfb4ae9d2319ac96004b37cc0283d06a2cd3b35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-7498493, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T11:01:43.033 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:42 smithi006 bash[93865]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3 2024-09-17T11:01:43.274 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:43 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:43.138+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:43.275 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:43 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:43.014+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:43.375 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:43 smithi006 podman[93883]: 2024-09-17 11:01:43.032408574 +0000 UTC m=+0.600242802 container remove 654bf1272724d1afe4d81d46fbfb4ae9d2319ac96004b37cc0283d06a2cd3b35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493) 2024-09-17T11:01:43.645 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:43 smithi006 podman[93969]: 2024-09-17 11:01:43.375121144 +0000 UTC m=+0.110064398 container create 013ca091aafd9768338ca15d9f938d49cba751e78a97af0935d81e7ddf7dc7b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-17T11:01:43.645 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:43 smithi006 podman[93969]: 2024-09-17 11:01:43.282137798 +0000 UTC m=+0.017081071 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T11:01:43.645 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:43 smithi006 podman[93969]: 2024-09-17 11:01:43.569323175 +0000 UTC m=+0.304266441 container init 013ca091aafd9768338ca15d9f938d49cba751e78a97af0935d81e7ddf7dc7b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-17T11:01:43.645 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:43 smithi006 podman[93969]: 2024-09-17 11:01:43.572909534 +0000 UTC m=+0.307852796 container start 013ca091aafd9768338ca15d9f938d49cba751e78a97af0935d81e7ddf7dc7b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T11:01:43.645 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:43 smithi006 podman[93969]: 2024-09-17 11:01:43.644131163 +0000 UTC m=+0.379074430 container attach 013ca091aafd9768338ca15d9f938d49cba751e78a97af0935d81e7ddf7dc7b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-7498493, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T11:01:43.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:43 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:43.392+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:43.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:43 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:43.273+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:43.951 INFO:journalctl@ceph.osd.3.smithi006.stdout:Sep 17 11:01:43 smithi006 podman[93969]: 2024-09-17 11:01:43.766818275 +0000 UTC m=+0.501761545 container died 013ca091aafd9768338ca15d9f938d49cba751e78a97af0935d81e7ddf7dc7b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-17T11:01:44.312 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:44 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:44.153+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:44.312 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:43 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:43.973+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:44.386 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.3.service' 2024-09-17T11:01:44.430 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:01:44.430 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-17T11:01:44.430 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-17T11:01:44.430 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.4 2024-09-17T11:01:44.706 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:44 smithi149 systemd[1]: Stopping Ceph osd.4 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:01:44.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:44 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:44.354+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:44.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:44 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:44.310+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:45.129 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:44 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:44.940+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:45.129 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:44 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:44.775+0000 7ff72c905640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:01:45.129 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:44 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:44.775+0000 7ff72c905640 -1 osd.4 157 *** Got signal Terminated *** 2024-09-17T11:01:45.129 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:44 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:44.775+0000 7ff72c905640 -1 osd.4 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T11:01:45.129 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:45 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:45.127+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:45.456 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:45 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:45.333+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:45.456 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:45 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:45.262+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:46.206 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:46 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:46.161+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:46.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:45 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:45.919+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:46.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:46 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:46.374+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:46.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:46 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:46.289+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:47.305 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:47 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:47.161+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:47.305 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:46 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:46.957+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:47.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:47 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:47.348+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:47.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:47 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:47.303+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:48.355 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:47 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:47.993+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:48.355 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:48 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:48.164+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:48.355 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:48 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:48.164+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:22.205005+0000 front 2024-09-17T11:01:22.204990+0000 (oldest deadline 2024-09-17T11:01:48.104867+0000) 2024-09-17T11:01:48.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:48 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:48.353+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:48.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:48 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:48.363+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:48.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:48 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:48.363+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:49.333 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:49.196+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:15.304001+0000 front 2024-09-17T11:01:15.303907+0000 (oldest deadline 2024-09-17T11:01:38.704161+0000) 2024-09-17T11:01:49.333 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4[53571]: 2024-09-17T11:01:49.196+0000 7ff724f12640 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:22.205005+0000 front 2024-09-17T11:01:22.204990+0000 (oldest deadline 2024-09-17T11:01:48.104867+0000) 2024-09-17T11:01:49.334 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:48 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:48.995+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:49.334 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:48 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:48.995+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:49.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:49.332+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:49.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:49.332+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:49.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:49.356+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:49.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:49.356+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:50.259 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:49.965+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:50.260 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:49 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:49.965+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:50.544 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:50 smithi149 podman[68676]: 2024-09-17 11:01:50.259124465 +0000 UTC m=+5.603606108 container died 016956823ff1173cf85e4c4cb00affa5c291b010d4737596f47c4f55c25697df (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-17T11:01:50.544 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:50 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:50.318+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:50.544 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:50 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:50.318+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:50.545 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:50 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:50.349+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:50.545 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:50 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:50.349+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:50.918 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:50 smithi149 podman[68676]: 2024-09-17 11:01:50.751289957 +0000 UTC m=+6.095771600 container cleanup 016956823ff1173cf85e4c4cb00affa5c291b010d4737596f47c4f55c25697df (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T11:01:50.919 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:50 smithi149 bash[68676]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4 2024-09-17T11:01:51.206 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:50 smithi149 podman[68693]: 2024-09-17 11:01:50.994567071 +0000 UTC m=+0.733397101 container remove 016956823ff1173cf85e4c4cb00affa5c291b010d4737596f47c4f55c25697df (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.license=GPLv2) 2024-09-17T11:01:51.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:50 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:50.917+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:51.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:50 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:50.917+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:51.590 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:51.346+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:51.590 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:51.346+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:51.590 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:51.314+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:51.590 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:51.314+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:51.591 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:51 smithi149 podman[68779]: 2024-09-17 11:01:51.231125183 +0000 UTC m=+0.016354135 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T11:01:51.591 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:51 smithi149 podman[68779]: 2024-09-17 11:01:51.353915653 +0000 UTC m=+0.139144586 container create f74af1c758e0dcc54495529a7d2b99564774dd50d0665ac8e88948faccfec37d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True) 2024-09-17T11:01:51.591 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:51 smithi149 podman[68779]: 2024-09-17 11:01:51.589396596 +0000 UTC m=+0.374625540 container init f74af1c758e0dcc54495529a7d2b99564774dd50d0665ac8e88948faccfec37d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-7498493, ceph=True) 2024-09-17T11:01:51.873 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:51 smithi149 podman[68779]: 2024-09-17 11:01:51.593023644 +0000 UTC m=+0.378252575 container start f74af1c758e0dcc54495529a7d2b99564774dd50d0665ac8e88948faccfec37d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-7498493) 2024-09-17T11:01:51.873 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:51 smithi149 podman[68779]: 2024-09-17 11:01:51.673067118 +0000 UTC m=+0.458296049 container attach f74af1c758e0dcc54495529a7d2b99564774dd50d0665ac8e88948faccfec37d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T11:01:51.873 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 17 11:01:51 smithi149 podman[68779]: 2024-09-17 11:01:51.782844165 +0000 UTC m=+0.568073116 container died f74af1c758e0dcc54495529a7d2b99564774dd50d0665ac8e88948faccfec37d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2) 2024-09-17T11:01:52.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:51.871+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:52.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:51 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:51.871+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:52.375 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.4.service' 2024-09-17T11:01:52.415 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:01:52.415 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-17T11:01:52.415 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-17T11:01:52.416 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.5 2024-09-17T11:01:52.634 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:52.325+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:52.634 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:52.325+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:52.634 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:52.369+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:52.634 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:52.369+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:52.634 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:52 smithi149 systemd[1]: Stopping Ceph osd.5 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:01:52.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:52.788+0000 7f48ba437640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:01:52.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:52.788+0000 7f48ba437640 -1 osd.5 157 *** Got signal Terminated *** 2024-09-17T11:01:52.956 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:52.788+0000 7f48ba437640 -1 osd.5 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T11:01:52.956 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:52.862+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:52.957 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:52 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:52.862+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:53.598 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:53 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:53.368+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:53.599 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:53 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:53.368+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:53.599 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:53 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:53.342+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:53.599 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:53 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:53.342+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:53.956 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:53 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:53.835+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:53.956 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:53 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:53.835+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:54.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:54 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:54.325+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:54.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:54 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:54.325+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:54.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:54 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:54.319+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:54.707 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:54 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:54.319+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:55.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:54 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:54.877+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:55.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:54 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:54.877+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:55.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:55.286+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:55.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:55.286+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:55.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:55.310+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:55.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:55.310+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:56.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:55.860+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:56.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:55.860+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:56.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:55 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:55.860+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:01:56.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:56.343+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:56.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:56.343+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:56.707 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:56.336+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:56.707 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:56.336+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:56.707 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:56.336+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.501265+0000 front 2024-09-17T11:01:32.501173+0000 (oldest deadline 2024-09-17T11:01:55.901597+0000) 2024-09-17T11:01:57.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:56.909+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:57.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:56.909+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:57.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:56 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:56.909+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:01:57.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:57.296+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:57.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:57.296+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:57.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:57.296+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:01:57.706 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:57.312+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:16.699641+0000 front 2024-09-17T11:01:16.699531+0000 (oldest deadline 2024-09-17T11:01:41.399470+0000) 2024-09-17T11:01:57.707 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:57.312+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:21.400050+0000 front 2024-09-17T11:01:21.400022+0000 (oldest deadline 2024-09-17T11:01:47.800288+0000) 2024-09-17T11:01:57.707 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5[55585]: 2024-09-17T11:01:57.312+0000 7f48b6a4c640 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.501265+0000 front 2024-09-17T11:01:32.501173+0000 (oldest deadline 2024-09-17T11:01:55.901597+0000) 2024-09-17T11:01:58.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:57.897+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:58.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:57.897+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:58.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:57 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:57.897+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:01:58.539 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:58 smithi149 podman[68898]: 2024-09-17 11:01:58.266833748 +0000 UTC m=+5.584816911 container died 144416f89b4a49ae129dafc8245910e18b49da7c741c87653ed754be4bc6ae9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-17T11:01:58.540 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:58.284+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:58.540 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:58.284+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:58.540 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:58.284+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:01:58.887 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:58 smithi149 podman[68898]: 2024-09-17 11:01:58.746870548 +0000 UTC m=+6.064853693 container cleanup 144416f89b4a49ae129dafc8245910e18b49da7c741c87653ed754be4bc6ae9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-17T11:01:58.887 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:58 smithi149 bash[68898]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5 2024-09-17T11:01:59.206 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:58 smithi149 podman[68924]: 2024-09-17 11:01:58.998166654 +0000 UTC m=+0.727480771 container remove 144416f89b4a49ae129dafc8245910e18b49da7c741c87653ed754be4bc6ae9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-7498493, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T11:01:59.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:58.886+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:01:59.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:58.886+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:01:59.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:58 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:58.886+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:01:59.544 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:59.250+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:01:59.544 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:59.250+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:01:59.544 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:01:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:01:59.250+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:01:59.544 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:59 smithi149 podman[69015]: 2024-09-17 11:01:59.226644585 +0000 UTC m=+0.015903438 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T11:01:59.544 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:59 smithi149 podman[69015]: 2024-09-17 11:01:59.340806769 +0000 UTC m=+0.130065611 container create 949fa26fae79651f8d77018e98076881b475ebee55149094f79fbd58dd8a70cb (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-17T11:01:59.544 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:59 smithi149 podman[69015]: 2024-09-17 11:01:59.543228016 +0000 UTC m=+0.332486864 container init 949fa26fae79651f8d77018e98076881b475ebee55149094f79fbd58dd8a70cb (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T11:01:59.932 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:59 smithi149 podman[69015]: 2024-09-17 11:01:59.546713749 +0000 UTC m=+0.335972594 container start 949fa26fae79651f8d77018e98076881b475ebee55149094f79fbd58dd8a70cb (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-7498493, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T11:01:59.932 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:59 smithi149 podman[69015]: 2024-09-17 11:01:59.643024202 +0000 UTC m=+0.432283060 container attach 949fa26fae79651f8d77018e98076881b475ebee55149094f79fbd58dd8a70cb (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T11:01:59.932 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 17 11:01:59 smithi149 podman[69015]: 2024-09-17 11:01:59.735198032 +0000 UTC m=+0.524456875 container died 949fa26fae79651f8d77018e98076881b475ebee55149094f79fbd58dd8a70cb (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=main-7498493, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T11:02:00.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:59.930+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:00.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:59.930+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:00.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:01:59 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:01:59.930+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:00.434 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.5.service' 2024-09-17T11:02:00.473 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:00.218+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:02:00.474 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:00.218+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:02:00.474 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:00.218+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:02:00.475 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:02:00.475 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-17T11:02:00.475 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-17T11:02:00.476 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.6 2024-09-17T11:02:00.810 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:00 smithi149 systemd[1]: Stopping Ceph osd.6 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:02:01.206 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:00.808+0000 7fc6a880f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:02:01.206 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:00.808+0000 7fc6a880f640 -1 osd.6 157 *** Got signal Terminated *** 2024-09-17T11:02:01.206 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:00.808+0000 7fc6a880f640 -1 osd.6 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T11:02:01.207 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:00.966+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:01.207 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:00.966+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:01.207 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:00 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:00.966+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:01.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:01 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:01.219+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:02:01.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:01 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:01.219+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:02:01.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:01 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:01.219+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:02:02.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:01 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:01.952+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:02.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:01 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:01.952+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:02.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:01 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:01.952+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:02.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:02 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:02.250+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:02:02.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:02 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:02.250+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:02:02.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:02 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:02.250+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:02:03.290 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:02 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:02.986+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:03.290 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:02 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:02.986+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:03.290 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:02 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:02.986+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:03.656 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:03.288+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:02:03.657 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:03.288+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:02:03.657 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:03.288+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:02:04.305 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:03.965+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:04.305 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:03.965+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:04.305 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:03 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:03.965+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:04.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:04.304+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:02:04.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:04.304+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:02:04.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:04.304+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:02:05.328 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:04.988+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:05.329 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:04.988+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:05.329 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:04.988+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:05.329 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:04 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:04.988+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:05.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:05.327+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.128978+0000 front 2024-09-17T11:01:14.129023+0000 (oldest deadline 2024-09-17T11:01:38.828934+0000) 2024-09-17T11:02:05.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:05.327+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:25.830445+0000 front 2024-09-17T11:01:25.830596+0000 (oldest deadline 2024-09-17T11:01:49.330394+0000) 2024-09-17T11:02:05.706 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6[57588]: 2024-09-17T11:02:05.327+0000 7fc6a0e1c640 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:32.831425+0000 front 2024-09-17T11:01:32.831401+0000 (oldest deadline 2024-09-17T11:01:56.931360+0000) 2024-09-17T11:02:06.299 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:05.993+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:06.299 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:05.993+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:06.299 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:05.993+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:06.299 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:05 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:05.993+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:06.576 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:06 smithi149 podman[69117]: 2024-09-17 11:02:06.297990599 +0000 UTC m=+5.600048546 container died 2908fd9a4510345ec32913f8c41c9abd3b251f6eae1c715b525e882528b7c40a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-17T11:02:06.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:06 smithi149 podman[69117]: 2024-09-17 11:02:06.74983452 +0000 UTC m=+6.051892467 container cleanup 2908fd9a4510345ec32913f8c41c9abd3b251f6eae1c715b525e882528b7c40a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T11:02:06.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:06 smithi149 bash[69117]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6 2024-09-17T11:02:07.361 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:07 smithi149 podman[69136]: 2024-09-17 11:02:07.017808996 +0000 UTC m=+0.722639528 container remove 2908fd9a4510345ec32913f8c41c9abd3b251f6eae1c715b525e882528b7c40a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-17T11:02:07.361 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:06 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:06.958+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:07.362 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:06 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:06.958+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:07.362 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:06 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:06.958+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:07.362 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:06 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:06.958+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:07.680 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:07 smithi149 podman[69222]: 2024-09-17 11:02:07.360377451 +0000 UTC m=+0.104934142 container create f6bbbbfb9f097328bf16dafcff72d8c8ffe752c4f7298880b9fc8184e7dadc47 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T11:02:07.680 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:07 smithi149 podman[69222]: 2024-09-17 11:02:07.27162252 +0000 UTC m=+0.016179177 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T11:02:07.681 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:07 smithi149 podman[69222]: 2024-09-17 11:02:07.604528426 +0000 UTC m=+0.349085072 container init f6bbbbfb9f097328bf16dafcff72d8c8ffe752c4f7298880b9fc8184e7dadc47 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-7498493) 2024-09-17T11:02:07.681 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:07 smithi149 podman[69222]: 2024-09-17 11:02:07.608426032 +0000 UTC m=+0.352982680 container start f6bbbbfb9f097328bf16dafcff72d8c8ffe752c4f7298880b9fc8184e7dadc47 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=main-7498493, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-17T11:02:07.681 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:07 smithi149 podman[69222]: 2024-09-17 11:02:07.679625309 +0000 UTC m=+0.424181966 container attach f6bbbbfb9f097328bf16dafcff72d8c8ffe752c4f7298880b9fc8184e7dadc47 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-7498493, GIT_BRANCH=HEAD) 2024-09-17T11:02:07.956 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 17 11:02:07 smithi149 podman[69222]: 2024-09-17 11:02:07.799819055 +0000 UTC m=+0.544375710 container died f6bbbbfb9f097328bf16dafcff72d8c8ffe752c4f7298880b9fc8184e7dadc47 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-7498493, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True) 2024-09-17T11:02:08.408 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:07 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:07.957+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:08.408 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:07 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:07.957+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:08.408 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:07 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:07.957+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:08.408 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:07 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:07.957+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:08.434 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.6.service' 2024-09-17T11:02:08.474 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:02:08.475 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-17T11:02:08.475 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-17T11:02:08.475 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.7 2024-09-17T11:02:08.706 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:08 smithi149 systemd[1]: Stopping Ceph osd.7 for ddf74116-74e0-11ef-bceb-c7b262605968... 2024-09-17T11:02:09.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:08 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:08.802+0000 7f9f24784640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:02:09.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:08 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:08.802+0000 7f9f24784640 -1 osd.7 157 *** Got signal Terminated *** 2024-09-17T11:02:09.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:08 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:08.802+0000 7f9f24784640 -1 osd.7 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T11:02:09.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:08 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:08.975+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:09.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:08 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:08.975+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:09.207 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:08 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:08.975+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:09.207 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:08 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:08.975+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:10.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:09 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:09.930+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:10.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:09 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:09.930+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:10.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:09 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:09.930+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:10.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:09 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:09.930+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:11.378 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:10 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:10.969+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:11.378 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:10 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:10.969+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:11.378 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:10 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:10.969+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:11.378 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:10 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:10.969+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:12.456 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:11 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:11.954+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:12.457 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:11 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:11.954+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:12.457 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:11 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:11.954+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:12.457 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:11 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:11.954+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:13.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:12 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:12.925+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-09-17T11:01:14.850010+0000 front 2024-09-17T11:01:14.849999+0000 (oldest deadline 2024-09-17T11:01:40.749901+0000) 2024-09-17T11:02:13.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:12 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:12.925+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-09-17T11:01:24.151703+0000 front 2024-09-17T11:01:24.151604+0000 (oldest deadline 2024-09-17T11:01:48.851572+0000) 2024-09-17T11:02:13.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:12 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:12.925+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-09-17T11:01:28.852157+0000 front 2024-09-17T11:01:28.852228+0000 (oldest deadline 2024-09-17T11:01:55.252282+0000) 2024-09-17T11:02:13.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:12 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:12.925+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-09-17T11:01:38.554184+0000 front 2024-09-17T11:01:38.554132+0000 (oldest deadline 2024-09-17T11:02:04.453936+0000) 2024-09-17T11:02:13.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:12 smithi149 ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7[59616]: 2024-09-17T11:02:12.925+0000 7f9f20d99640 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.149:6804 osd.4 since back 2024-09-17T11:01:48.955494+0000 front 2024-09-17T11:01:48.955455+0000 (oldest deadline 2024-09-17T11:02:12.455492+0000) 2024-09-17T11:02:14.469 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:14 smithi149 podman[69325]: 2024-09-17 11:02:14.206861346 +0000 UTC m=+5.501316036 container died 8f7c760b4de3c7de3dec920d445c5487214b3dc1bcdcf2034055009f2491e591 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-17T11:02:14.820 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:14 smithi149 podman[69325]: 2024-09-17 11:02:14.643638924 +0000 UTC m=+5.938093615 container cleanup 8f7c760b4de3c7de3dec920d445c5487214b3dc1bcdcf2034055009f2491e591 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-7498493, ceph=True) 2024-09-17T11:02:14.821 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:14 smithi149 bash[69325]: ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7 2024-09-17T11:02:15.206 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:14 smithi149 podman[69368]: 2024-09-17 11:02:14.81988131 +0000 UTC m=+0.606925889 container remove 8f7c760b4de3c7de3dec920d445c5487214b3dc1bcdcf2034055009f2491e591 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T11:02:15.557 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:15 smithi149 podman[69459]: 2024-09-17 11:02:15.151695669 +0000 UTC m=+0.016054371 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T11:02:15.557 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:15 smithi149 podman[69459]: 2024-09-17 11:02:15.254310096 +0000 UTC m=+0.118668788 container create 4c48f102e3d0c1bc65902a65fec9e9d0c3b72816e06b2ed0dc7ccd6893b64fda (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T11:02:15.557 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:15 smithi149 podman[69459]: 2024-09-17 11:02:15.473524999 +0000 UTC m=+0.337883688 container init 4c48f102e3d0c1bc65902a65fec9e9d0c3b72816e06b2ed0dc7ccd6893b64fda (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-7498493, org.label-schema.build-date=20240731) 2024-09-17T11:02:15.557 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:15 smithi149 podman[69459]: 2024-09-17 11:02:15.476875284 +0000 UTC m=+0.341233973 container start 4c48f102e3d0c1bc65902a65fec9e9d0c3b72816e06b2ed0dc7ccd6893b64fda (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-7498493, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T11:02:15.557 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:15 smithi149 podman[69459]: 2024-09-17 11:02:15.556436288 +0000 UTC m=+0.420794984 container attach 4c48f102e3d0c1bc65902a65fec9e9d0c3b72816e06b2ed0dc7ccd6893b64fda (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-17T11:02:15.956 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 17 11:02:15 smithi149 podman[69459]: 2024-09-17 11:02:15.667699583 +0000 UTC m=+0.532058275 container died 4c48f102e3d0c1bc65902a65fec9e9d0c3b72816e06b2ed0dc7ccd6893b64fda (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-ddf74116-74e0-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T11:02:16.231 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ddf74116-74e0-11ef-bceb-c7b262605968@osd.7.service' 2024-09-17T11:02:16.276 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T11:02:16.276 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-17T11:02:16.276 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ddf74116-74e0-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T11:05:13.260 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ddf74116-74e0-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T11:05:32.879 DEBUG:teuthology.orchestra.run.smithi006:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T11:05:32.914 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T11:05:32.941 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-17T11:05:32.942 DEBUG:teuthology.misc:Transferring archived files from smithi006:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690/remote/smithi006/crash 2024-09-17T11:05:32.943 DEBUG:teuthology.orchestra.run.smithi006:> sudo tar c -f - -C /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/crash -- . 2024-09-17T11:05:32.996 INFO:teuthology.orchestra.run.smithi006.stderr:tar: /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T11:05:32.996 INFO:teuthology.orchestra.run.smithi006.stderr:tar: Error is not recoverable: exiting now 2024-09-17T11:05:32.998 DEBUG:teuthology.misc:Transferring archived files from smithi149:/var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690/remote/smithi149/crash 2024-09-17T11:05:32.999 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/crash -- . 2024-09-17T11:05:33.036 INFO:teuthology.orchestra.run.smithi149.stderr:tar: /var/lib/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T11:05:33.036 INFO:teuthology.orchestra.run.smithi149.stderr:tar: Error is not recoverable: exiting now 2024-09-17T11:05:33.037 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-17T11:05:33.037 DEBUG:teuthology.orchestra.run.smithi006:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-17T11:05:33.071 INFO:tasks.cephadm:Compressing logs... 2024-09-17T11:05:33.071 DEBUG:teuthology.orchestra.run.smithi006:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T11:05:33.114 DEBUG:teuthology.orchestra.run.smithi149:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T11:05:33.140 INFO:teuthology.orchestra.run.smithi006.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-17T11:05:33.140 INFO:teuthology.orchestra.run.smithi006.stderr:: No such file or directory 2024-09-17T11:05:33.140 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-17T11:05:33.140 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T11:05:33.141 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mon.smithi006.log 2024-09-17T11:05:33.141 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T11:05:33.143 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mon.smithi006.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi006.nripkg.log 2024-09-17T11:05:33.143 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T11:05:33.143 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi006.nripkg.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-17T11:05:33.143 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mgr.smithi006.plhvdm.log 2024-09-17T11:05:33.144 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.2.log: 85.7% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T11:05:33.144 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.log 2024-09-17T11:05:33.147 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mgr.smithi006.plhvdm.log: 91.4% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T11:05:33.148 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-17T11:05:33.150 INFO:teuthology.orchestra.run.smithi149.stderr:find: ‘/var/log/rbd-target-api’gzip: No such file or directory -5 2024-09-17T11:05:33.150 INFO:teuthology.orchestra.run.smithi149.stderr: --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-17T11:05:33.151 INFO:teuthology.orchestra.run.smithi149.stderr:gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T11:05:33.152 INFO:teuthology.orchestra.run.smithi149.stderr:gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-17T11:05:33.152 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.7.log: /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T11:05:33.152 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mgr.smithi149.ijckag.log 2024-09-17T11:05:33.152 INFO:teuthology.orchestra.run.smithi149.stderr:gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mon.smithi149.log 2024-09-17T11:05:33.153 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.log: 87.0% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T11:05:33.153 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T11:05:33.155 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.audit.log: /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mgr.smithi149.ijckag.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T11:05:33.155 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mon.smithi149.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-17T11:05:33.155 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-17T11:05:33.155 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.log 2024-09-17T11:05:33.155 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi149.hmuwiq.log 2024-09-17T11:05:33.155 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T11:05:33.155 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi149.hmuwiq.log: /var/log/ceph/cephadm.log: 87.5% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi149.hmuwiq.log.gz 2024-09-17T11:05:33.156 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.3.log: 91.2% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T11:05:33.169 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi006.nripkg.log.gz 2024-09-17T11:05:33.188 INFO:teuthology.orchestra.run.smithi006.stderr: 91.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T11:05:33.218 INFO:teuthology.orchestra.run.smithi149.stderr: 85.7% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T11:05:33.293 INFO:teuthology.orchestra.run.smithi149.stderr: 87.0% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T11:05:33.296 INFO:teuthology.orchestra.run.smithi149.stderr: 91.6% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T11:05:33.405 INFO:teuthology.orchestra.run.smithi149.stderr: 91.2% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T11:05:33.538 INFO:teuthology.orchestra.run.smithi006.stderr: 88.6% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mgr.smithi006.plhvdm.log.gz 2024-09-17T11:05:33.695 INFO:teuthology.orchestra.run.smithi149.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T11:05:34.153 INFO:teuthology.orchestra.run.smithi149.stderr: 89.8% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mgr.smithi149.ijckag.log.gz 2024-09-17T11:05:34.463 INFO:teuthology.orchestra.run.smithi006.stderr: 91.8% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mon.smithi006.log.gz 2024-09-17T11:05:37.724 INFO:teuthology.orchestra.run.smithi149.stderr: 92.6% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-mon.smithi149.log.gz 2024-09-17T11:05:47.243 INFO:teuthology.orchestra.run.smithi006.stderr: 94.5% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-17T11:05:49.395 INFO:teuthology.orchestra.run.smithi006.stderr: 94.1% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-17T11:05:50.273 INFO:teuthology.orchestra.run.smithi149.stderr: 93.9% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-17T11:05:50.799 INFO:teuthology.orchestra.run.smithi006.stderr: 94.0% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-17T11:05:51.602 INFO:teuthology.orchestra.run.smithi006.stderr: 94.2% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-17T11:05:51.605 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-09-17T11:05:51.605 INFO:teuthology.orchestra.run.smithi006.stderr:real 0m18.476s 2024-09-17T11:05:51.605 INFO:teuthology.orchestra.run.smithi006.stderr:user 1m5.759s 2024-09-17T11:05:51.605 INFO:teuthology.orchestra.run.smithi006.stderr:sys 0m2.395s 2024-09-17T11:05:53.094 INFO:teuthology.orchestra.run.smithi149.stderr: 94.2% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-17T11:05:57.182 INFO:teuthology.orchestra.run.smithi149.stderr: 94.2% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-17T11:05:58.315 INFO:teuthology.orchestra.run.smithi149.stderr: 94.0% -- replaced with /var/log/ceph/ddf74116-74e0-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-17T11:05:58.318 INFO:teuthology.orchestra.run.smithi149.stderr: 2024-09-17T11:05:58.318 INFO:teuthology.orchestra.run.smithi149.stderr:real 0m25.189s 2024-09-17T11:05:58.318 INFO:teuthology.orchestra.run.smithi149.stderr:user 1m6.244s 2024-09-17T11:05:58.319 INFO:teuthology.orchestra.run.smithi149.stderr:sys 0m2.761s 2024-09-17T11:05:58.319 INFO:tasks.cephadm:Archiving logs... 2024-09-17T11:05:58.320 DEBUG:teuthology.misc:Transferring archived files from smithi006:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690/remote/smithi006/log 2024-09-17T11:05:58.321 DEBUG:teuthology.orchestra.run.smithi006:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T11:06:07.509 DEBUG:teuthology.misc:Transferring archived files from smithi149:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690/remote/smithi149/log 2024-09-17T11:06:07.511 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T11:06:16.932 INFO:tasks.cephadm:Removing cluster... 2024-09-17T11:06:16.933 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ddf74116-74e0-11ef-bceb-c7b262605968 --force 2024-09-17T11:06:17.307 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ddf74116-74e0-11ef-bceb-c7b262605968 --force 2024-09-17T11:06:17.649 INFO:tasks.cephadm:Removing cephadm ... 2024-09-17T11:06:17.649 DEBUG:teuthology.orchestra.run.smithi006:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T11:06:17.666 DEBUG:teuthology.orchestra.run.smithi149:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T11:06:17.681 INFO:tasks.cephadm:Teardown complete 2024-09-17T11:06:17.682 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-17T11:06:17.691 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-17T11:06:17.692 DEBUG:teuthology.orchestra.run.smithi006:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T11:06:17.709 DEBUG:teuthology.orchestra.run.smithi149:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T11:06:17.721 INFO:teuthology.orchestra.run.smithi006.stderr:bash: line 1: ntpq: command not found 2024-09-17T11:06:17.726 INFO:teuthology.orchestra.run.smithi006.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T11:06:17.727 INFO:teuthology.orchestra.run.smithi006.stdout:=============================================================================== 2024-09-17T11:06:17.727 INFO:teuthology.orchestra.run.smithi006.stdout:^* hv01.front.sepia.ceph.com 3 6 377 26 -529us[ -494us] +/- 46ms 2024-09-17T11:06:17.727 INFO:teuthology.orchestra.run.smithi006.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 27 +797us[ +832us] +/- 67ms 2024-09-17T11:06:17.727 INFO:teuthology.orchestra.run.smithi006.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 27 +55us[ +90us] +/- 86ms 2024-09-17T11:06:17.727 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:06:17.736 INFO:teuthology.orchestra.run.smithi149.stderr:bash: line 1: ntpq: command not found 2024-09-17T11:06:17.750 INFO:teuthology.orchestra.run.smithi149.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T11:06:17.751 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================== 2024-09-17T11:06:17.751 INFO:teuthology.orchestra.run.smithi149.stdout:^* hv01.front.sepia.ceph.com 3 6 377 26 -522us[ -459us] +/- 46ms 2024-09-17T11:06:17.751 INFO:teuthology.orchestra.run.smithi149.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 26 +776us[ +776us] +/- 67ms 2024-09-17T11:06:17.751 INFO:teuthology.orchestra.run.smithi149.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 28 +50us[ +113us] +/- 86ms 2024-09-17T11:06:17.751 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:06:17.751 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-17T11:06:17.761 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-17T11:06:17.761 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-17T11:06:17.772 DEBUG:teuthology.orchestra.run.smithi006:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T11:06:17.852 DEBUG:teuthology.orchestra.run.smithi149:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T11:06:17.968 DEBUG:teuthology.orchestra.run.smithi006:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T11:06:17.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T11:06:17.998 DEBUG:teuthology.orchestra.run.smithi149:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T11:06:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T11:06:18.025 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-17T11:06:18.035 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-17T11:06:18.044 INFO:teuthology.task.internal:Duration was 1952.546060 seconds 2024-09-17T11:06:18.044 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-17T11:06:18.053 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-17T11:06:18.053 DEBUG:teuthology.orchestra.run.smithi006:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T11:06:18.056 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T11:06:18.093 INFO:teuthology.orchestra.run.smithi006.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T11:06:18.111 INFO:teuthology.orchestra.run.smithi149.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T11:06:18.490 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-17T11:06:18.490 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi006.front.sepia.ceph.com 2024-09-17T11:06:18.490 DEBUG:teuthology.orchestra.run.smithi006:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T11:06:18.552 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi149.front.sepia.ceph.com 2024-09-17T11:06:18.552 DEBUG:teuthology.orchestra.run.smithi149:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T11:06:18.576 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-17T11:06:18.576 DEBUG:teuthology.orchestra.run.smithi006:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T11:06:18.595 DEBUG:teuthology.orchestra.run.smithi149:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T11:06:18.643 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-17T11:06:18.643 DEBUG:teuthology.orchestra.run.smithi006:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T11:06:18.662 DEBUG:teuthology.orchestra.run.smithi149:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T11:06:33.152 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-17T11:06:33.166 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-17T11:06:33.166 DEBUG:teuthology.orchestra.run.smithi006:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T11:06:33.192 DEBUG:teuthology.orchestra.run.smithi149:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T11:06:33.224 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-17T11:06:33.234 DEBUG:teuthology.orchestra.run.smithi006:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T11:06:33.236 DEBUG:teuthology.orchestra.run.smithi149:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T11:06:33.258 INFO:teuthology.orchestra.run.smithi006.stdout:kernel.core_pattern = core 2024-09-17T11:06:33.290 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern = core 2024-09-17T11:06:33.309 DEBUG:teuthology.orchestra.run.smithi006:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T11:06:33.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T11:06:33.341 DEBUG:teuthology.orchestra.run.smithi149:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T11:06:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T11:06:33.364 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-17T11:06:33.376 INFO:teuthology.task.internal:Transferring archived files... 2024-09-17T11:06:33.377 DEBUG:teuthology.misc:Transferring archived files from smithi006:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690/remote/smithi006 2024-09-17T11:06:33.377 DEBUG:teuthology.orchestra.run.smithi006:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T11:06:33.443 DEBUG:teuthology.misc:Transferring archived files from smithi149:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907690/remote/smithi149 2024-09-17T11:06:33.444 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T11:06:33.484 INFO:teuthology.task.internal:Removing archive directory... 2024-09-17T11:06:33.484 DEBUG:teuthology.orchestra.run.smithi006:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T11:06:33.487 DEBUG:teuthology.orchestra.run.smithi149:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T11:06:33.536 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-17T11:06:33.548 INFO:teuthology.task.internal:Not uploading archives. 2024-09-17T11:06:33.548 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-17T11:06:33.558 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-17T11:06:33.558 DEBUG:teuthology.orchestra.run.smithi006:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T11:06:33.561 DEBUG:teuthology.orchestra.run.smithi149:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T11:06:33.574 INFO:teuthology.orchestra.run.smithi006.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 11:06 /home/ubuntu/cephtest 2024-09-17T11:06:33.591 INFO:teuthology.orchestra.run.smithi149.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 11:06 /home/ubuntu/cephtest 2024-09-17T11:06:33.605 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-17T11:06:33.616 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-17T11:06:33.641 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1952.5460600852966 owner: scheduled_teuthology@teuthology success: true 2024-09-17T11:06:33.641 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T11:06:33.721 INFO:teuthology.run:pass