2024-03-30T13:06:22.048 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-30T13:06:22.049 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077 --verbose --description orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077/orig.config.yaml 2024-03-30T13:06:22.087 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-30T13:06:22.168 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7630077' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 4e50bbcdd1630c7760b708f3a5b4d38a121e66cb ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 4e50bbcdd1630c7760b708f3a5b4d38a121e66cb selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: 5af0400050a123dc31f105425174f106e057276a owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 8777 sha1: 4e50bbcdd1630c7760b708f3a5b4d38a121e66cb sleep_before_teardown: 0 subset: 43/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_5af0400050a123dc31f105425174f106e057276a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 5af0400050a123dc31f105425174f106e057276a targets: smithi159.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAhS3uAOwmqwZkto2LEa9WB2bREDcFW5GXpupBhN9iSSKjQfNgtY6ZWb94PW8qsfSftV1I1158op8sF2x5hjh6U= smithi187.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPGpbOiGjH1fjHnJig6Re17NRUTfgotqtnTfySHE65gHF2Tjvv8k8orLSTil90JtgqeJi+2x8FKG4kseo4KP9Fg= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-29_21:08:02 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-03-30T13:06:22.169 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_5af0400050a123dc31f105425174f106e057276a/qa; will attempt to use it 2024-03-30T13:06:22.169 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_5af0400050a123dc31f105425174f106e057276a/qa/tasks 2024-03-30T13:06:22.169 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-30T13:06:22.171 INFO:teuthology.task.internal:Checking packages... 2024-03-30T13:06:22.197 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '4e50bbcdd1630c7760b708f3a5b4d38a121e66cb' 2024-03-30T13:06:22.198 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-30T13:06:22.198 INFO:teuthology.packaging:ref: None 2024-03-30T13:06:22.198 INFO:teuthology.packaging:tag: None 2024-03-30T13:06:22.198 INFO:teuthology.packaging:branch: squid 2024-03-30T13:06:22.198 INFO:teuthology.packaging:sha1: 4e50bbcdd1630c7760b708f3a5b4d38a121e66cb 2024-03-30T13:06:22.198 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-03-30T13:06:22.373 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1973.g5af04000 2024-03-30T13:06:22.375 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-30T13:06:22.380 INFO:teuthology.task.internal:no buildpackages task found 2024-03-30T13:06:22.380 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-30T13:06:22.496 INFO:teuthology.task.internal:Saving configuration 2024-03-30T13:06:22.510 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-30T13:06:22.513 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-30T13:06:22.539 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi159.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-30 13:01:13.794067', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAhS3uAOwmqwZkto2LEa9WB2bREDcFW5GXpupBhN9iSSKjQfNgtY6ZWb94PW8qsfSftV1I1158op8sF2x5hjh6U='} 2024-03-30T13:06:22.563 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi187.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-30 13:01:13.795325', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPGpbOiGjH1fjHnJig6Re17NRUTfgotqtnTfySHE65gHF2Tjvv8k8orLSTil90JtgqeJi+2x8FKG4kseo4KP9Fg='} 2024-03-30T13:06:22.563 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-30T13:06:22.568 INFO:teuthology.task.internal:roles: ubuntu@smithi159.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-03-30T13:06:22.568 INFO:teuthology.task.internal:roles: ubuntu@smithi187.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-03-30T13:06:22.568 INFO:teuthology.run_tasks:Running task console_log... 2024-03-30T13:06:22.643 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f31a9b93f70>, signals=[15]) 2024-03-30T13:06:22.643 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-30T13:06:22.649 INFO:teuthology.task.internal:Opening connections... 2024-03-30T13:06:22.649 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:06:22.652 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:06:22.741 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi187.front.sepia.ceph.com 2024-03-30T13:06:22.742 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:06:22.831 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-30T13:06:22.837 DEBUG:teuthology.orchestra.run.smithi159:> uname -m 2024-03-30T13:06:22.855 INFO:teuthology.orchestra.run.smithi159.stdout:x86_64 2024-03-30T13:06:22.855 DEBUG:teuthology.orchestra.run.smithi159:> cat /etc/os-release 2024-03-30T13:06:22.913 INFO:teuthology.orchestra.run.smithi159.stdout:NAME="CentOS Stream" 2024-03-30T13:06:22.913 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION="9" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:ID="centos" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:ID_LIKE="rhel fedora" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION_ID="9" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:PLATFORM_ID="platform:el9" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:ANSI_COLOR="0;31" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:LOGO="fedora-logo-icon" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:HOME_URL="https://centos.org/" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-30T13:06:22.914 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-30T13:06:22.915 INFO:teuthology.lock.ops:Updating smithi159.front.sepia.ceph.com on lock server 2024-03-30T13:06:22.936 DEBUG:teuthology.orchestra.run.smithi187:> uname -m 2024-03-30T13:06:22.953 INFO:teuthology.orchestra.run.smithi187.stdout:x86_64 2024-03-30T13:06:22.953 DEBUG:teuthology.orchestra.run.smithi187:> cat /etc/os-release 2024-03-30T13:06:23.010 INFO:teuthology.orchestra.run.smithi187.stdout:NAME="CentOS Stream" 2024-03-30T13:06:23.010 INFO:teuthology.orchestra.run.smithi187.stdout:VERSION="9" 2024-03-30T13:06:23.010 INFO:teuthology.orchestra.run.smithi187.stdout:ID="centos" 2024-03-30T13:06:23.010 INFO:teuthology.orchestra.run.smithi187.stdout:ID_LIKE="rhel fedora" 2024-03-30T13:06:23.010 INFO:teuthology.orchestra.run.smithi187.stdout:VERSION_ID="9" 2024-03-30T13:06:23.010 INFO:teuthology.orchestra.run.smithi187.stdout:PLATFORM_ID="platform:el9" 2024-03-30T13:06:23.011 INFO:teuthology.orchestra.run.smithi187.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-30T13:06:23.011 INFO:teuthology.orchestra.run.smithi187.stdout:ANSI_COLOR="0;31" 2024-03-30T13:06:23.011 INFO:teuthology.orchestra.run.smithi187.stdout:LOGO="fedora-logo-icon" 2024-03-30T13:06:23.011 INFO:teuthology.orchestra.run.smithi187.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-30T13:06:23.011 INFO:teuthology.orchestra.run.smithi187.stdout:HOME_URL="https://centos.org/" 2024-03-30T13:06:23.011 INFO:teuthology.orchestra.run.smithi187.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-30T13:06:23.011 INFO:teuthology.orchestra.run.smithi187.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-30T13:06:23.011 INFO:teuthology.orchestra.run.smithi187.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-30T13:06:23.011 INFO:teuthology.lock.ops:Updating smithi187.front.sepia.ceph.com on lock server 2024-03-30T13:06:23.036 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-30T13:06:23.045 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-30T13:06:23.127 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-30T13:06:23.128 DEBUG:teuthology.orchestra.run.smithi159:> test '!' -e /home/ubuntu/cephtest 2024-03-30T13:06:23.130 DEBUG:teuthology.orchestra.run.smithi187:> test '!' -e /home/ubuntu/cephtest 2024-03-30T13:06:23.146 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-30T13:06:23.218 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-30T13:06:23.218 DEBUG:teuthology.orchestra.run.smithi159:> test -z $(ls -A /var/lib/ceph) 2024-03-30T13:06:23.220 DEBUG:teuthology.orchestra.run.smithi187:> test -z $(ls -A /var/lib/ceph) 2024-03-30T13:06:23.331 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-30T13:06:23.387 INFO:teuthology.run_tasks:Running task kernel... 2024-03-30T13:06:23.406 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-30T13:06:23.406 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-30T13:06:23.406 DEBUG:teuthology.orchestra.run.smithi159:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-30T13:06:23.407 DEBUG:teuthology.orchestra.run.smithi187:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-30T13:06:23.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-30T13:06:23.424 DEBUG:teuthology.orchestra.run.smithi159:> uname -r 2024-03-30T13:06:23.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-30T13:06:23.426 DEBUG:teuthology.orchestra.run.smithi187:> uname -r 2024-03-30T13:06:23.481 INFO:teuthology.orchestra.run.smithi159.stdout:5.14.0-361.el9.x86_64 2024-03-30T13:06:23.481 INFO:teuthology.task.kernel:Running kernel on smithi159: 5.14.0-361.el9.x86_64 2024-03-30T13:06:23.481 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum install -y kernel 2024-03-30T13:06:23.482 INFO:teuthology.orchestra.run.smithi187.stdout:5.14.0-361.el9.x86_64 2024-03-30T13:06:23.483 INFO:teuthology.task.kernel:Running kernel on smithi187: 5.14.0-361.el9.x86_64 2024-03-30T13:06:23.483 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum install -y kernel 2024-03-30T13:06:25.767 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-30T13:06:25.794 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-30T13:06:27.394 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS-9 - BaseOS 3.7 MB/s | 2.0 MB 00:00 2024-03-30T13:06:27.400 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS-9 - BaseOS 3.9 MB/s | 2.0 MB 00:00 2024-03-30T13:06:28.243 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.0 MB 00:00 2024-03-30T13:06:28.524 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.0 MB 00:00 2024-03-30T13:06:30.417 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - AppStream 23 MB/s | 19 MB 00:00 2024-03-30T13:06:33.069 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - AppStream 5.0 MB/s | 19 MB 00:03 2024-03-30T13:06:35.286 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - CRB 4.5 MB/s | 6.1 MB 00:01 2024-03-30T13:06:36.918 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - Extras packages 31 kB/s | 16 kB 00:00 2024-03-30T13:06:36.937 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - CRB 17 MB/s | 6.1 MB 00:00 2024-03-30T13:06:38.229 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - Extras packages 98 kB/s | 16 kB 00:00 2024-03-30T13:06:38.538 INFO:teuthology.orchestra.run.smithi187.stdout:Extra Packages for Enterprise Linux 14 MB/s | 21 MB 00:01 2024-03-30T13:06:39.133 INFO:teuthology.orchestra.run.smithi159.stdout:Extra Packages for Enterprise Linux 27 MB/s | 21 MB 00:00 2024-03-30T13:06:43.528 INFO:teuthology.orchestra.run.smithi187.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-03-30T13:06:44.072 INFO:teuthology.orchestra.run.smithi159.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-03-30T13:06:45.759 INFO:teuthology.orchestra.run.smithi187.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-30T13:06:45.759 INFO:teuthology.orchestra.run.smithi187.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-30T13:06:45.851 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-30T13:06:45.856 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-30T13:06:45.857 INFO:teuthology.orchestra.run.smithi187.stdout:Install 4 Packages 2024-03-30T13:06:45.858 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:06:45.858 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 76 M 2024-03-30T13:06:45.858 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 123 M 2024-03-30T13:06:45.858 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-30T13:06:46.129 INFO:teuthology.orchestra.run.smithi187.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 709 kB/s | 192 kB 00:00 2024-03-30T13:06:46.321 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-30T13:06:46.321 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-30T13:06:46.410 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-03-30T13:06:46.416 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-03-30T13:06:46.416 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-03-30T13:06:46.416 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout:Install 4 Packages 2024-03-30T13:06:46.417 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:06:46.418 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 76 M 2024-03-30T13:06:46.418 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 123 M 2024-03-30T13:06:46.418 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-03-30T13:06:46.840 INFO:teuthology.orchestra.run.smithi159.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 455 kB/s | 192 kB 00:00 2024-03-30T13:06:47.012 INFO:teuthology.orchestra.run.smithi187.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 13 MB/s | 15 MB 00:01 2024-03-30T13:06:47.873 INFO:teuthology.orchestra.run.smithi159.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 10 MB/s | 15 MB 00:01 2024-03-30T13:06:47.990 INFO:teuthology.orchestra.run.smithi187.stdout:(3/4): kernel-modules-core-5.14.0-434.el9.x86_6 15 MB/s | 28 MB 00:01 2024-03-30T13:06:48.412 INFO:teuthology.orchestra.run.smithi187.stdout:(4/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 13 MB/s | 34 MB 00:02 2024-03-30T13:06:48.415 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-30T13:06:48.416 INFO:teuthology.orchestra.run.smithi187.stdout:Total 30 MB/s | 76 MB 00:02 2024-03-30T13:06:48.416 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-30T13:06:48.507 INFO:teuthology.orchestra.run.smithi159.stdout:(3/4): kernel-modules-core-5.14.0-434.el9.x86_6 17 MB/s | 28 MB 00:01 2024-03-30T13:06:48.783 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-30T13:06:48.783 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-30T13:06:48.982 INFO:teuthology.orchestra.run.smithi159.stdout:(4/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 13 MB/s | 34 MB 00:02 2024-03-30T13:06:48.985 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-03-30T13:06:48.986 INFO:teuthology.orchestra.run.smithi159.stdout:Total 30 MB/s | 76 MB 00:02 2024-03-30T13:06:48.986 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-03-30T13:06:49.371 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-03-30T13:06:49.371 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-03-30T13:06:49.618 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-30T13:06:49.618 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-30T13:06:50.211 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-03-30T13:06:50.212 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-03-30T13:06:51.925 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-30T13:06:52.495 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-03-30T13:06:52.532 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-03-30T13:06:52.621 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-30T13:06:53.146 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-03-30T13:06:53.241 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-30T13:06:55.381 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-30T13:06:55.514 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-30T13:06:55.993 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-30T13:06:56.134 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-30T13:07:00.214 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-30T13:07:00.253 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:00.834 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-30T13:07:00.876 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:04.975 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:05.598 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:34.448 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:34.449 INFO:teuthology.orchestra.run.smithi187.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-30T13:07:34.449 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:07:34.491 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:34.875 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:34.876 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-03-30T13:07:34.876 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-30T13:07:34.876 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-30T13:07:35.138 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:35.138 INFO:teuthology.orchestra.run.smithi159.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-30T13:07:35.138 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:07:35.181 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:35.289 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:35.289 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:07:35.289 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-30T13:07:35.289 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.289 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.289 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.289 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.289 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:07:35.290 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-30T13:07:35.514 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:35.514 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-03-30T13:07:35.514 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-30T13:07:35.514 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-30T13:07:35.622 DEBUG:teuthology.orchestra.run.smithi187:> rpm -q kernel | sort -rV | head -n 1 2024-03-30T13:07:35.724 INFO:teuthology.orchestra.run.smithi187.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.724 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-30T13:07:35.724 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-03-30T13:07:35.725 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-30T13:07:35.725 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-30T13:07:35.725 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-03-30T13:07:35.725 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi187.front.sepia.ceph.com, path=None, version=distro) 2024-03-30T13:07:35.725 DEBUG:teuthology.orchestra.run.smithi187:> rpm -q kernel | sort -rV | head -n 1 2024-03-30T13:07:35.804 INFO:teuthology.orchestra.run.smithi187.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.805 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-30T13:07:35.805 DEBUG:teuthology.orchestra.run.smithi187:> sudo rpm -qi grub2-tools 2024-03-30T13:07:35.840 INFO:teuthology.orchestra.run.smithi187.stdout:Name : grub2-tools 2024-03-30T13:07:35.840 INFO:teuthology.orchestra.run.smithi187.stdout:Epoch : 1 2024-03-30T13:07:35.840 INFO:teuthology.orchestra.run.smithi187.stdout:Version : 2.06 2024-03-30T13:07:35.840 INFO:teuthology.orchestra.run.smithi187.stdout:Release : 68.el9 2024-03-30T13:07:35.840 INFO:teuthology.orchestra.run.smithi187.stdout:Architecture: x86_64 2024-03-30T13:07:35.840 INFO:teuthology.orchestra.run.smithi187.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-30T13:07:35.840 INFO:teuthology.orchestra.run.smithi187.stdout:Group : Unspecified 2024-03-30T13:07:35.840 INFO:teuthology.orchestra.run.smithi187.stdout:Size : 8264254 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:License : GPLv3+ 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:Packager : builder@centos.org 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:Vendor : CentOS 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:Summary : Support tools for GRUB. 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:Description : 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-30T13:07:35.841 INFO:teuthology.orchestra.run.smithi187.stdout:hardware devices. 2024-03-30T13:07:35.842 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:07:35.842 INFO:teuthology.orchestra.run.smithi187.stdout:This subpackage provides tools for support of all platforms. 2024-03-30T13:07:35.842 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-30T13:07:35.843 INFO:teuthology.task.kernel:Updating grub on smithi187 to boot 5.14.0-434.el9.x86_64 2024-03-30T13:07:35.843 DEBUG:teuthology.orchestra.run.smithi187:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:07:35.929 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-03-30T13:07:36.251 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel | sort -rV | head -n 1 2024-03-30T13:07:36.357 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-30T13:07:36.357 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-30T13:07:36.357 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-03-30T13:07:36.358 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-30T13:07:36.358 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-30T13:07:36.358 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-03-30T13:07:36.358 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi159.front.sepia.ceph.com, path=None, version=distro) 2024-03-30T13:07:36.358 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel | sort -rV | head -n 1 2024-03-30T13:07:36.437 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-30T13:07:36.438 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-30T13:07:36.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo rpm -qi grub2-tools 2024-03-30T13:07:36.441 INFO:teuthology.orchestra.run.smithi187.stderr:Generating grub configuration file ... 2024-03-30T13:07:36.481 INFO:teuthology.orchestra.run.smithi159.stdout:Name : grub2-tools 2024-03-30T13:07:36.481 INFO:teuthology.orchestra.run.smithi159.stdout:Epoch : 1 2024-03-30T13:07:36.481 INFO:teuthology.orchestra.run.smithi159.stdout:Version : 2.06 2024-03-30T13:07:36.481 INFO:teuthology.orchestra.run.smithi159.stdout:Release : 68.el9 2024-03-30T13:07:36.481 INFO:teuthology.orchestra.run.smithi159.stdout:Architecture: x86_64 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Group : Unspecified 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Size : 8264254 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:License : GPLv3+ 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Packager : builder@centos.org 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Vendor : CentOS 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Summary : Support tools for GRUB. 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout:Description : 2024-03-30T13:07:36.482 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:07:36.483 INFO:teuthology.orchestra.run.smithi159.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-30T13:07:36.483 INFO:teuthology.orchestra.run.smithi159.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-30T13:07:36.483 INFO:teuthology.orchestra.run.smithi159.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-30T13:07:36.483 INFO:teuthology.orchestra.run.smithi159.stdout:hardware devices. 2024-03-30T13:07:36.483 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:07:36.483 INFO:teuthology.orchestra.run.smithi159.stdout:This subpackage provides tools for support of all platforms. 2024-03-30T13:07:36.484 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-30T13:07:36.484 INFO:teuthology.task.kernel:Updating grub on smithi159 to boot 5.14.0-434.el9.x86_64 2024-03-30T13:07:36.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-30T13:07:37.059 INFO:teuthology.orchestra.run.smithi159.stderr:Generating grub configuration file ... 2024-03-30T13:07:37.796 INFO:teuthology.orchestra.run.smithi187.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-30T13:07:37.849 INFO:teuthology.orchestra.run.smithi187.stderr:done 2024-03-30T13:07:37.851 DEBUG:teuthology.orchestra.run.smithi187:> mktemp 2024-03-30T13:07:37.868 INFO:teuthology.orchestra.run.smithi187.stdout:/tmp/tmp.xAtktM4glt 2024-03-30T13:07:37.869 DEBUG:teuthology.orchestra.run.smithi187:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.xAtktM4glt 2024-03-30T13:07:37.940 DEBUG:teuthology.orchestra.run.smithi187:> sudo chmod 0666 /tmp/tmp.xAtktM4glt 2024-03-30T13:07:38.084 DEBUG:teuthology.orchestra.remote:smithi187:/tmp/tmp.xAtktM4glt is 6KB 2024-03-30T13:07:38.132 DEBUG:teuthology.orchestra.run.smithi187:> rm -fr /tmp/tmp.xAtktM4glt 2024-03-30T13:07:38.146 DEBUG:teuthology.orchestra.run.smithi187:> sudo /bin/ls /boot/loader/entries || true 2024-03-30T13:07:38.211 INFO:teuthology.orchestra.run.smithi187.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-30T13:07:38.211 INFO:teuthology.orchestra.run.smithi187.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-30T13:07:38.211 INFO:teuthology.orchestra.run.smithi187.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-30T13:07:38.211 INFO:teuthology.orchestra.run.smithi187.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-03-30T13:07:38.212 DEBUG:teuthology.orchestra.run.smithi187:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-03-30T13:07:38.353 DEBUG:teuthology.orchestra.run.smithi187:> sudo shutdown -r now 2024-03-30T13:07:38.504 INFO:teuthology.orchestra.run.smithi159.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-30T13:07:38.533 INFO:teuthology.orchestra.run.smithi159.stderr:done 2024-03-30T13:07:38.535 DEBUG:teuthology.orchestra.run.smithi159:> mktemp 2024-03-30T13:07:38.550 INFO:teuthology.orchestra.run.smithi159.stdout:/tmp/tmp.dq73gJ8ctc 2024-03-30T13:07:38.551 DEBUG:teuthology.orchestra.run.smithi159:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.dq73gJ8ctc 2024-03-30T13:07:38.619 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 0666 /tmp/tmp.dq73gJ8ctc 2024-03-30T13:07:38.754 DEBUG:teuthology.orchestra.remote:smithi159:/tmp/tmp.dq73gJ8ctc is 6KB 2024-03-30T13:07:38.802 DEBUG:teuthology.orchestra.run.smithi159:> rm -fr /tmp/tmp.dq73gJ8ctc 2024-03-30T13:07:38.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo /bin/ls /boot/loader/entries || true 2024-03-30T13:07:38.886 INFO:teuthology.orchestra.run.smithi159.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-30T13:07:38.886 INFO:teuthology.orchestra.run.smithi159.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-30T13:07:38.886 INFO:teuthology.orchestra.run.smithi159.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-30T13:07:38.887 INFO:teuthology.orchestra.run.smithi159.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-03-30T13:07:38.888 DEBUG:teuthology.orchestra.run.smithi159:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-03-30T13:07:39.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo shutdown -r now 2024-03-30T13:08:08.382 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-03-30T13:08:08.409 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-30T13:08:08.410 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:08:09.047 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-03-30T13:08:09.047 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-03-30T13:08:09.047 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:08:43.683 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159 2024-03-30T13:08:52.690 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-03-30T13:08:52.691 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:09:08.426 DEBUG:teuthology.orchestra.remote:timed out 2024-03-30T13:09:11.070 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159 2024-03-30T13:09:17.434 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-30T13:09:17.435 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:09:17.864 DEBUG:teuthology.orchestra.run.smithi187:> true 2024-03-30T13:09:18.278 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-30T13:09:18.278 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-434.el9.x86_64"... 2024-03-30T13:09:18.279 DEBUG:teuthology.orchestra.run.smithi187:> uname -r 2024-03-30T13:09:18.293 INFO:teuthology.orchestra.run.smithi187.stdout:5.14.0-434.el9.x86_64 2024-03-30T13:09:18.293 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-03-30T13:09:18.293 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-30T13:09:18.293 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-30T13:09:19.294 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-03-30T13:09:19.294 DEBUG:teuthology.orchestra.run.smithi187:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-30T13:09:19.343 INFO:teuthology.orchestra.run.smithi187.stdout:ttyS1 2024-03-30T13:09:19.395 DEBUG:teuthology.parallel:result is None 2024-03-30T13:09:23.071 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-03-30T13:09:23.072 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:09:23.400 DEBUG:teuthology.orchestra.run.smithi159:> true 2024-03-30T13:09:23.975 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-03-30T13:09:23.975 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-434.el9.x86_64"... 2024-03-30T13:09:23.975 DEBUG:teuthology.orchestra.run.smithi159:> uname -r 2024-03-30T13:09:23.990 INFO:teuthology.orchestra.run.smithi159.stdout:5.14.0-434.el9.x86_64 2024-03-30T13:09:23.990 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-03-30T13:09:23.990 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-30T13:09:23.990 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-30T13:09:24.991 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-03-30T13:09:24.991 DEBUG:teuthology.orchestra.run.smithi159:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-30T13:09:25.088 INFO:teuthology.orchestra.run.smithi159.stdout:ttyS1 2024-03-30T13:09:25.122 DEBUG:teuthology.parallel:result is None 2024-03-30T13:09:25.122 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-30T13:09:25.130 INFO:teuthology.task.internal:Creating test directory... 2024-03-30T13:09:25.130 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-30T13:09:25.132 DEBUG:teuthology.orchestra.run.smithi187:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-30T13:09:25.150 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-30T13:09:25.157 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-30T13:09:25.170 INFO:teuthology.task.internal:Creating archive directory... 2024-03-30T13:09:25.170 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-30T13:09:25.188 DEBUG:teuthology.orchestra.run.smithi187:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-30T13:09:25.227 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-30T13:09:25.234 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-30T13:09:25.235 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-30T13:09:25.247 DEBUG:teuthology.orchestra.run.smithi187:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-30T13:09:25.274 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-30T13:09:25.288 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-30T13:09:25.303 INFO:teuthology.orchestra.run.smithi187.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-30T13:09:25.317 INFO:teuthology.orchestra.run.smithi187.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-30T13:09:25.319 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-30T13:09:25.326 INFO:teuthology.task.internal:Configuring sudo... 2024-03-30T13:09:25.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-30T13:09:25.333 DEBUG:teuthology.orchestra.run.smithi187:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-30T13:09:25.389 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-30T13:09:25.398 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-30T13:09:25.398 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-30T13:09:25.402 DEBUG:teuthology.orchestra.run.smithi187:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-30T13:09:25.446 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-30T13:09:25.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-30T13:09:25.542 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-30T13:09:25.622 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-30T13:09:25.667 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:09:25.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-30T13:09:25.746 DEBUG:teuthology.orchestra.run.smithi187:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-30T13:09:25.787 DEBUG:teuthology.orchestra.run.smithi187:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-30T13:09:25.847 DEBUG:teuthology.orchestra.run.smithi187:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-30T13:09:25.933 DEBUG:teuthology.orchestra.run.smithi187:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-30T13:09:25.979 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-30T13:09:25.979 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-30T13:09:26.050 DEBUG:teuthology.orchestra.run.smithi159:> sudo service rsyslog restart 2024-03-30T13:09:26.053 DEBUG:teuthology.orchestra.run.smithi187:> sudo service rsyslog restart 2024-03-30T13:09:26.100 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-30T13:09:26.148 INFO:teuthology.orchestra.run.smithi187.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-30T13:09:26.565 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-30T13:09:26.573 INFO:teuthology.task.internal:Starting timer... 2024-03-30T13:09:26.573 INFO:teuthology.run_tasks:Running task pcp... 2024-03-30T13:09:26.584 INFO:teuthology.run_tasks:Running task selinux... 2024-03-30T13:09:26.593 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-03-30T13:09:26.593 DEBUG:teuthology.orchestra.run.smithi159:> sudo service auditd rotate 2024-03-30T13:09:26.650 INFO:teuthology.orchestra.run.smithi159.stdout:Rotating logs: 2024-03-30T13:09:26.652 DEBUG:teuthology.orchestra.run.smithi187:> sudo service auditd rotate 2024-03-30T13:09:26.717 INFO:teuthology.orchestra.run.smithi187.stdout:Rotating logs: 2024-03-30T13:09:26.719 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-30T13:09:26.719 DEBUG:teuthology.orchestra.run.smithi159:> /usr/sbin/getenforce 2024-03-30T13:09:26.738 INFO:teuthology.orchestra.run.smithi159.stdout:Permissive 2024-03-30T13:09:26.739 DEBUG:teuthology.orchestra.run.smithi187:> /usr/sbin/getenforce 2024-03-30T13:09:26.759 INFO:teuthology.orchestra.run.smithi187.stdout:Permissive 2024-03-30T13:09:26.759 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi159.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi187.front.sepia.ceph.com': 'permissive'} 2024-03-30T13:09:26.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-30T13:09:26.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-30T13:09:26.809 DEBUG:teuthology.orchestra.run.smithi187:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-30T13:09:26.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-30T13:09:26.839 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-30T13:09:26.839 DEBUG:teuthology.orchestra.run.smithi159:> sudo /usr/sbin/setenforce permissive 2024-03-30T13:09:26.890 DEBUG:teuthology.orchestra.run.smithi187:> sudo /usr/sbin/setenforce permissive 2024-03-30T13:09:26.926 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-30T13:09:26.936 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-30T13:09:26.954 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-30T13:09:26.955 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi159.front.sepia.ceph.com,smithi187.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-30T13:15:21.757 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi159.front.sepia.ceph.com'), Remote(name='ubuntu@smithi187.front.sepia.ceph.com')] 2024-03-30T13:15:21.758 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-03-30T13:15:21.760 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:15:21.849 DEBUG:teuthology.orchestra.run.smithi159:> true 2024-03-30T13:15:21.929 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-03-30T13:15:21.929 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-30T13:15:21.930 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-30T13:15:22.018 DEBUG:teuthology.orchestra.run.smithi187:> true 2024-03-30T13:15:22.100 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-30T13:15:22.101 INFO:teuthology.run_tasks:Running task clock... 2024-03-30T13:15:22.113 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-30T13:15:22.113 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-30T13:15:22.114 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-30T13:15:22.116 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-30T13:15:22.116 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-30T13:15:22.145 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-30T13:15:22.162 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-30T13:15:22.171 INFO:teuthology.orchestra.run.smithi187.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-30T13:15:22.187 INFO:teuthology.orchestra.run.smithi187.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-30T13:15:22.190 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: ntpd: command not found 2024-03-30T13:15:22.204 INFO:teuthology.orchestra.run.smithi159.stdout:506 Cannot talk to daemon 2024-03-30T13:15:22.216 INFO:teuthology.orchestra.run.smithi187.stderr:sudo: ntpd: command not found 2024-03-30T13:15:22.221 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-30T13:15:22.230 INFO:teuthology.orchestra.run.smithi187.stdout:506 Cannot talk to daemon 2024-03-30T13:15:22.237 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-30T13:15:22.246 INFO:teuthology.orchestra.run.smithi187.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-30T13:15:22.262 INFO:teuthology.orchestra.run.smithi187.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-30T13:15:22.295 INFO:teuthology.orchestra.run.smithi159.stderr:bash: line 1: ntpq: command not found 2024-03-30T13:15:22.298 INFO:teuthology.orchestra.run.smithi159.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-30T13:15:22.298 INFO:teuthology.orchestra.run.smithi159.stdout:=============================================================================== 2024-03-30T13:15:22.298 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:15:22.298 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:15:22.298 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:15:22.298 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:15:22.315 INFO:teuthology.orchestra.run.smithi187.stderr:bash: line 1: ntpq: command not found 2024-03-30T13:15:22.318 INFO:teuthology.orchestra.run.smithi187.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-30T13:15:22.318 INFO:teuthology.orchestra.run.smithi187.stdout:=============================================================================== 2024-03-30T13:15:22.319 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:15:22.319 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:15:22.319 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:15:22.319 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:15:22.320 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-30T13:15:22.421 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '4e50bbcdd1630c7760b708f3a5b4d38a121e66cb'} 2024-03-30T13:15:22.422 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-03-30T13:15:22.425 INFO:tasks.cephadm:Cluster fsid is 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:15:22.425 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-30T13:15:22.425 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-03-30T13:15:22.425 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi159': '172.21.15.159', 'mon.smithi187': '172.21.15.187'} 2024-03-30T13:15:22.425 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-30T13:15:22.426 DEBUG:teuthology.orchestra.run.smithi159:> sudo hostname $(hostname -s) 2024-03-30T13:15:22.454 DEBUG:teuthology.orchestra.run.smithi187:> sudo hostname $(hostname -s) 2024-03-30T13:15:22.482 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-03-30T13:15:22.482 DEBUG:teuthology.orchestra.run.smithi159:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-30T13:15:22.743 INFO:teuthology.orchestra.run.smithi159.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Mar 30 13:15 /home/ubuntu/cephtest/cephadm 2024-03-30T13:15:22.743 DEBUG:teuthology.orchestra.run.smithi187:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-30T13:15:23.020 INFO:teuthology.orchestra.run.smithi187.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Mar 30 13:15 /home/ubuntu/cephtest/cephadm 2024-03-30T13:15:23.020 DEBUG:teuthology.orchestra.run.smithi159:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-30T13:15:23.036 DEBUG:teuthology.orchestra.run.smithi187:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-30T13:15:23.069 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-03-30T13:15:23.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-03-30T13:15:23.078 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-03-30T13:15:23.839 INFO:teuthology.orchestra.run.smithi187.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-30T13:15:23.966 INFO:teuthology.orchestra.run.smithi159.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-30T13:15:41.114 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:15:41.114 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-03-30T13:15:41.114 INFO:teuthology.orchestra.run.smithi159.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-03-30T13:15:41.114 INFO:teuthology.orchestra.run.smithi159.stdout: "repo_digests": [ 2024-03-30T13:15:41.115 INFO:teuthology.orchestra.run.smithi159.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-03-30T13:15:41.115 INFO:teuthology.orchestra.run.smithi159.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-03-30T13:15:41.115 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-03-30T13:15:41.115 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:15:44.884 INFO:teuthology.orchestra.run.smithi187.stdout:{ 2024-03-30T13:15:44.884 INFO:teuthology.orchestra.run.smithi187.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-03-30T13:15:44.884 INFO:teuthology.orchestra.run.smithi187.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-03-30T13:15:44.884 INFO:teuthology.orchestra.run.smithi187.stdout: "repo_digests": [ 2024-03-30T13:15:44.884 INFO:teuthology.orchestra.run.smithi187.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-03-30T13:15:44.884 INFO:teuthology.orchestra.run.smithi187.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-03-30T13:15:44.884 INFO:teuthology.orchestra.run.smithi187.stdout: ] 2024-03-30T13:15:44.884 INFO:teuthology.orchestra.run.smithi187.stdout:} 2024-03-30T13:15:44.903 DEBUG:teuthology.orchestra.run.smithi159:> sudo mkdir -p /etc/ceph 2024-03-30T13:15:44.939 DEBUG:teuthology.orchestra.run.smithi187:> sudo mkdir -p /etc/ceph 2024-03-30T13:15:44.970 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 777 /etc/ceph 2024-03-30T13:15:45.012 DEBUG:teuthology.orchestra.run.smithi187:> sudo chmod 777 /etc/ceph 2024-03-30T13:15:45.039 INFO:tasks.cephadm:Writing seed config... 2024-03-30T13:15:45.040 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-30T13:15:45.040 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-30T13:15:45.040 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-30T13:15:45.040 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-30T13:15:45.040 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-30T13:15:45.041 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-30T13:15:45.041 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-30T13:15:45.041 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-30T13:15:45.041 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:15:45.041 DEBUG:teuthology.orchestra.run.smithi159:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-30T13:15:45.069 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9058771a-ee97-11ee-b647-cb9ed24678a4 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-30T13:15:45.070 DEBUG:teuthology.orchestra.run.smithi159:mon.smithi159> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159.service 2024-03-30T13:15:45.112 INFO:tasks.cephadm:Bootstrapping... 2024-03-30T13:15:45.112 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.159 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-30T13:15:45.312 INFO:teuthology.orchestra.run.smithi159.stderr:-------------------------------------------------------------------------------- 2024-03-30T13:15:45.312 INFO:teuthology.orchestra.run.smithi159.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '9058771a-ee97-11ee-b647-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.159', '--skip-admin-label'] 2024-03-30T13:15:45.335 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: 4.9.4-dev 2024-03-30T13:15:45.337 INFO:teuthology.orchestra.run.smithi159.stderr:Verifying podman|docker is present... 2024-03-30T13:15:45.358 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: 4.9.4-dev 2024-03-30T13:15:45.361 INFO:teuthology.orchestra.run.smithi159.stderr:Verifying lvm2 is present... 2024-03-30T13:15:45.361 INFO:teuthology.orchestra.run.smithi159.stderr:Verifying time synchronization is in place... 2024-03-30T13:15:45.367 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-03-30T13:15:45.373 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: inactive 2024-03-30T13:15:45.380 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: enabled 2024-03-30T13:15:45.386 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: active 2024-03-30T13:15:45.386 INFO:teuthology.orchestra.run.smithi159.stderr:Unit chronyd.service is enabled and running 2024-03-30T13:15:45.386 INFO:teuthology.orchestra.run.smithi159.stderr:Repeating the final host check... 2024-03-30T13:15:45.407 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: 4.9.4-dev 2024-03-30T13:15:45.409 INFO:teuthology.orchestra.run.smithi159.stderr:podman (/bin/podman) version 4.9.4 is present 2024-03-30T13:15:45.409 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl is present 2024-03-30T13:15:45.409 INFO:teuthology.orchestra.run.smithi159.stderr:lvcreate is present 2024-03-30T13:15:45.415 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-03-30T13:15:45.421 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: inactive 2024-03-30T13:15:45.427 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: enabled 2024-03-30T13:15:45.433 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: active 2024-03-30T13:15:45.434 INFO:teuthology.orchestra.run.smithi159.stderr:Unit chronyd.service is enabled and running 2024-03-30T13:15:45.434 INFO:teuthology.orchestra.run.smithi159.stderr:Host looks OK 2024-03-30T13:15:45.434 INFO:teuthology.orchestra.run.smithi159.stderr:Cluster fsid: 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:15:45.434 INFO:teuthology.orchestra.run.smithi159.stderr:Acquiring lock 140088750858592 on /run/cephadm/9058771a-ee97-11ee-b647-cb9ed24678a4.lock 2024-03-30T13:15:45.434 INFO:teuthology.orchestra.run.smithi159.stderr:Lock 140088750858592 acquired on /run/cephadm/9058771a-ee97-11ee-b647-cb9ed24678a4.lock 2024-03-30T13:15:45.435 INFO:teuthology.orchestra.run.smithi159.stderr:Verifying IP 172.21.15.159 port 3300 ... 2024-03-30T13:15:45.435 INFO:teuthology.orchestra.run.smithi159.stderr:Verifying IP 172.21.15.159 port 6789 ... 2024-03-30T13:15:45.435 INFO:teuthology.orchestra.run.smithi159.stderr:Base mon IP is 172.21.15.159, final addrv is [v2:172.21.15.159:3300,v1:172.21.15.159:6789] 2024-03-30T13:15:45.437 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.159 metric 100 2024-03-30T13:15:45.437 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.159 metric 100 2024-03-30T13:15:45.439 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-03-30T13:15:45.440 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-03-30T13:15:45.440 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1777sec hoplimit 64 pref medium 2024-03-30T13:15:45.443 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-30T13:15:45.443 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-03-30T13:15:45.443 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-03-30T13:15:45.443 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-03-30T13:15:45.443 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:724f/64 scope link 2024-03-30T13:15:45.443 INFO:teuthology.orchestra.run.smithi159.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-03-30T13:15:45.444 INFO:teuthology.orchestra.run.smithi159.stderr:Mon IP `172.21.15.159` is in CIDR network `172.21.0.0/20` 2024-03-30T13:15:45.444 INFO:teuthology.orchestra.run.smithi159.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-30T13:15:45.444 INFO:teuthology.orchestra.run.smithi159.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-30T13:15:45.544 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-03-30T13:15:45.960 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Getting image source signatures 2024-03-30T13:15:45.960 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-03-30T13:15:45.960 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-03-30T13:15:45.960 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-03-30T13:15:45.960 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-03-30T13:15:45.960 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-03-30T13:15:45.960 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-03-30T13:15:46.084 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: Writing manifest to image destination 2024-03-30T13:15:46.194 INFO:teuthology.orchestra.run.smithi159.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-03-30T13:15:46.554 INFO:teuthology.orchestra.run.smithi159.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-03-30T13:15:47.068 INFO:teuthology.orchestra.run.smithi159.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-03-30T13:15:47.068 INFO:teuthology.orchestra.run.smithi159.stderr:Extracting ceph user uid/gid from container image... 2024-03-30T13:15:47.443 INFO:teuthology.orchestra.run.smithi159.stderr:stat: 167 167 2024-03-30T13:15:47.882 INFO:teuthology.orchestra.run.smithi159.stderr:Creating initial keys... 2024-03-30T13:15:48.323 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph-authtool: AQCEEAhmsXkQDhAAUntjt8aid0//5IinDjcaxg== 2024-03-30T13:15:49.228 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph-authtool: AQCFEAhmN6nkChAASkgxaqK3xaBGNL0uTCDJ3g== 2024-03-30T13:15:50.392 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph-authtool: AQCGEAhmJJozExAAwZXk9Dn3nhU8vuDJ6dtAaw== 2024-03-30T13:15:50.882 INFO:teuthology.orchestra.run.smithi159.stderr:Creating initial monmap... 2024-03-30T13:15:51.231 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-30T13:15:51.231 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-03-30T13:15:51.231 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:15:51.231 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-30T13:15:51.770 INFO:teuthology.orchestra.run.smithi159.stderr:monmaptool for smithi159 [v2:172.21.15.159:3300,v1:172.21.15.159:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-30T13:15:51.771 INFO:teuthology.orchestra.run.smithi159.stderr:setting min_mon_release = octopus 2024-03-30T13:15:51.771 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/monmaptool: set fsid to 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:15:51.771 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-30T13:15:51.771 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:15:51.771 INFO:teuthology.orchestra.run.smithi159.stderr:Creating mon... 2024-03-30T13:15:52.684 INFO:teuthology.orchestra.run.smithi159.stderr:create mon.smithi159 on 2024-03-30T13:15:52.901 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-03-30T13:15:53.046 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4.target. 2024-03-30T13:15:53.046 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4.target. 2024-03-30T13:15:53.340 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Failed to reset failed state of unit ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159.service: Unit ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159.service not loaded. 2024-03-30T13:15:53.348 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4.target.wants/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159.service → /etc/systemd/system/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@.service. 2024-03-30T13:15:54.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:53 smithi159 ceph-mon[24443]: mon.smithi159 is new leader, mons smithi159 in quorum (ranks 0) 2024-03-30T13:15:54.033 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: disabled 2024-03-30T13:15:54.040 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: inactive 2024-03-30T13:15:54.040 INFO:teuthology.orchestra.run.smithi159.stderr:firewalld.service is not enabled 2024-03-30T13:15:54.040 INFO:teuthology.orchestra.run.smithi159.stderr:Not possible to enable service . firewalld.service is not available 2024-03-30T13:15:54.041 INFO:teuthology.orchestra.run.smithi159.stderr:Waiting for mon to start... 2024-03-30T13:15:54.041 INFO:teuthology.orchestra.run.smithi159.stderr:Waiting for mon... 2024-03-30T13:15:54.322 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:54 smithi159 bash[24425]: 7f03c5b9de3cfede46197863b7832036d11cb08eef0db229b28f989fedf07c73 2024-03-30T13:15:54.322 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:54 smithi159 systemd[1]: Started Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: cluster: 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: id: 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: services: 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi159 (age 0.521942s) 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mgr: no daemons active 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: data: 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-03-30T13:15:54.497 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-03-30T13:15:54.498 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-03-30T13:15:54.498 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: pgs: 2024-03-30T13:15:54.498 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:15:55.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:54 smithi159 ceph-mon[24443]: mon.smithi159 is new leader, mons smithi159 in quorum (ranks 0) 2024-03-30T13:15:55.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:54 smithi159 ceph-mon[24443]: monmap e1: 1 mons at {smithi159=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0]} 2024-03-30T13:15:55.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:54 smithi159 ceph-mon[24443]: fsmap 2024-03-30T13:15:55.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:54 smithi159 ceph-mon[24443]: osdmap e1: 0 total, 0 up, 0 in 2024-03-30T13:15:55.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:54 smithi159 ceph-mon[24443]: mgrmap e1: no daemons active 2024-03-30T13:15:55.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:54 smithi159 ceph-mon[24443]: from='client.? 172.21.15.159:0/1240217293' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-30T13:15:55.102 INFO:teuthology.orchestra.run.smithi159.stderr:mon is available 2024-03-30T13:15:55.102 INFO:teuthology.orchestra.run.smithi159.stderr:Assimilating anything we can from ceph.conf... 2024-03-30T13:15:55.604 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:15:55.604 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: [global] 2024-03-30T13:15:55.604 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: fsid = 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.159:3300,v1:172.21.15.159:6789] 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: [mgr] 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: [osd] 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-03-30T13:15:55.605 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-03-30T13:15:56.163 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:55 smithi159 ceph-mon[24443]: from='client.? 172.21.15.159:0/3902968569' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-30T13:15:56.163 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:55 smithi159 ceph-mon[24443]: from='client.? 172.21.15.159:0/3902968569' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-30T13:15:56.165 INFO:teuthology.orchestra.run.smithi159.stderr:Generating new minimal ceph.conf... 2024-03-30T13:15:57.072 INFO:teuthology.orchestra.run.smithi159.stderr:Restarting the monitor... 2024-03-30T13:15:57.214 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:56 smithi159 ceph-mon[24443]: from='client.? 172.21.15.159:0/786236936' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:15:57.214 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:57 smithi159 systemd[1]: Stopping Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:15:57.214 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:57 smithi159 bash[24661]: Error: no container with name or ID "ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon.smithi159" found: no such container 2024-03-30T13:15:57.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:57 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[24439]: 2024-03-30T13:15:57.209+0000 7fd86dd0a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi159 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:15:57.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:57 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[24439]: 2024-03-30T13:15:57.209+0000 7fd86dd0a700 -1 mon.smithi159@0(leader) e1 *** Got Signal Terminated *** 2024-03-30T13:15:57.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:57 smithi159 podman[24675]: 2024-03-30 13:15:57.404722438 +0000 UTC m=+0.282987768 container died 7f03c5b9de3cfede46197863b7832036d11cb08eef0db229b28f989fedf07c73 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, RELEASE=HEAD, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-30T13:15:58.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:57 smithi159 podman[24675]: 2024-03-30 13:15:57.792954745 +0000 UTC m=+0.671220109 container cleanup 7f03c5b9de3cfede46197863b7832036d11cb08eef0db229b28f989fedf07c73 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.openshift.expose-services=, io.buildah.version=1.19.8) 2024-03-30T13:15:58.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:57 smithi159 bash[24675]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159 2024-03-30T13:15:58.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:57 smithi159 bash[24702]: Error: no container with name or ID "ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon.smithi159" found: no such container 2024-03-30T13:15:58.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 podman[24689]: 2024-03-30 13:15:58.00190684 +0000 UTC m=+0.595944804 container remove 7f03c5b9de3cfede46197863b7832036d11cb08eef0db229b28f989fedf07c73 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, release=754, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.component=centos-stream-container, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_CLEAN=True) 2024-03-30T13:15:58.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159.service: Deactivated successfully. 2024-03-30T13:15:58.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 systemd[1]: Stopped Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:15:58.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 systemd[1]: Starting Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:15:58.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 podman[24768]: 2024-03-30 13:15:58.194311073 +0000 UTC m=+0.087068209 container create d58042490a7e0511e9cbe78be6c4f13b4bd32a4e5b2791c1600fa136ef0e6f3c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, release=754, architecture=x86_64, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, RELEASE=HEAD, GIT_CLEAN=True, version=8, io.openshift.tags=base centos centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-30T13:15:58.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 podman[24768]: 2024-03-30 13:15:58.125805327 +0000 UTC m=+0.018562420 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-03-30T13:15:58.453 INFO:teuthology.orchestra.run.smithi159.stderr:Setting mon public_network to 172.21.0.0/20 2024-03-30T13:15:58.579 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 podman[24768]: 2024-03-30 13:15:58.363846842 +0000 UTC m=+0.256603946 container init d58042490a7e0511e9cbe78be6c4f13b4bd32a4e5b2791c1600fa136ef0e6f3c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, io.k8s.display-name=CentOS Stream 8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, ceph=True, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-03-30T13:15:58.579 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 podman[24768]: 2024-03-30 13:15:58.366877944 +0000 UTC m=+0.259635039 container start d58042490a7e0511e9cbe78be6c4f13b4bd32a4e5b2791c1600fa136ef0e6f3c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, io.openshift.expose-services=, name=centos-stream, release=754, architecture=x86_64, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-03-30T13:15:58.579 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: set uid:gid to 167:167 (ceph:ceph) 2024-03-30T13:15:58.579 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-03-30T13:15:58.579 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: pidfile_write: ignore empty --pid-file 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: load: jerasure load: lrc 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: RocksDB version: 6.15.5 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Compile date Apr 18 2022 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: DB SUMMARY 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: DB Session ID: T79LOVQD4ERXS2R57070 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: CURRENT file: CURRENT 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: IDENTITY file: IDENTITY 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi159/store.db dir, Total Num: 1, files: 000008.sst 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi159/store.db: 000010.log size: 82739 ; 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.error_if_exists: 0 2024-03-30T13:15:58.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.create_if_missing: 0 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.paranoid_checks: 1 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.env: 0x55a4394c6860 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.fs: Posix File System 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.info_log: 0x55a43b0bbfa0 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_file_opening_threads: 16 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.statistics: (nil) 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.use_fsync: 0 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_log_file_size: 0 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.keep_log_file_num: 1000 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.recycle_log_file_num: 0 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.allow_fallocate: 1 2024-03-30T13:15:58.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.allow_mmap_reads: 0 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.allow_mmap_writes: 0 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.use_direct_reads: 0 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.create_missing_column_families: 0 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.db_log_dir: 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi159/store.db 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-30T13:15:58.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.advise_random_on_open: 1 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.db_write_buffer_size: 0 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.write_buffer_manager: 0x55a43b1ac2d0 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.rate_limiter: (nil) 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.wal_recovery_mode: 2 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.enable_thread_tracking: 0 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.enable_pipelined_write: 0 2024-03-30T13:15:58.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.unordered_write: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.row_cache: None 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.wal_filter: None 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.allow_ingest_behind: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.preserve_deletes: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.two_write_queues: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.manual_wal_flush: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.atomic_flush: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-30T13:15:58.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.log_readahead_size: 0 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.best_efforts_recovery: 0 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.allow_data_in_errors: 0 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.db_host_id: __hostname__ 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_background_jobs: 2 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_background_compactions: -1 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_subcompactions: 1 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-30T13:15:58.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_total_wal_size: 0 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_open_files: -1 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bytes_per_sync: 0 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_readahead_size: 0 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_background_flushes: -1 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Compression algorithms supported: 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: kZSTD supported: 0 2024-03-30T13:15:58.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: kXpressCompression supported: 0 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: kLZ4HCCompression supported: 1 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: kLZ4Compression supported: 1 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: kBZip2Compression supported: 0 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: kZlibCompression supported: 1 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: kSnappyCompression supported: 1 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi159/store.db/MANIFEST-000009 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.merge_operator: 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_filter: None 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_filter_factory: None 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.sst_partitioner_factory: None 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-30T13:15:58.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a43b087d00) 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: cache_index_and_filter_blocks: 1 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: pin_top_level_index_and_filter: 1 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: index_type: 0 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: data_block_index_type: 0 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: index_shortening: 1 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: hash_index_allow_collision: 1 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: checksum: 1 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: no_block_cache: 0 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_cache: 0x55a43b0f2170 2024-03-30T13:15:58.589 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_cache_name: BinnedLRUCache 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_cache_options: 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: capacity : 536870912 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: num_shard_bits : 4 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: strict_capacity_limit : 0 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: high_pri_pool_ratio: 0.000 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_cache_compressed: (nil) 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: persistent_cache: (nil) 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_size: 4096 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_size_deviation: 10 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_restart_interval: 16 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: index_block_restart_interval: 1 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: metadata_block_size: 4096 2024-03-30T13:15:58.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: partition_filters: 0 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: use_delta_encoding: 1 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: whole_key_filtering: 1 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: verify_compression: 0 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: read_amp_bytes_per_bit: 0 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: format_version: 4 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: enable_index_compression: 1 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_align: 0 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.write_buffer_size: 33554432 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_write_buffer_number: 2 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compression: NoCompression 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bottommost_compression: Disabled 2024-03-30T13:15:58.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.prefix_extractor: nullptr 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.num_levels: 7 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compression_opts.level: 32767 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compression_opts.strategy: 0 2024-03-30T13:15:58.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-30T13:15:58.593 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-30T13:15:58.593 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-30T13:15:58.593 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compression_opts.enabled: false 2024-03-30T13:15:58.593 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-30T13:15:58.593 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-30T13:15:58.593 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.target_file_size_base: 67108864 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-30T13:15:58.594 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.arena_block_size: 4194304 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.disable_auto_compactions: 0 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-30T13:15:58.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.table_properties_collectors: 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.inplace_update_support: 0 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.bloom_locality: 0 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.max_successive_merges: 0 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.paranoid_file_checks: 0 2024-03-30T13:15:58.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.force_consistency_checks: 1 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.report_bg_io_stats: 0 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.ttl: 2592000 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.enable_blob_files: false 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.min_blob_size: 0 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.blob_file_size: 268435456 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi159/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711804558393173, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-03-30T13:15:58.597 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711804558394177, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79421, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77685, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71994, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711804558, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5ab294a9-fc7b-43c2-9a8b-95dbeeceea3d", "db_session_id": "T79LOVQD4ERXS2R57070"}} 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711804558394638, "job": 1, "event": "recovery_finished"} 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi159/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55a43b0d8a80 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: DB pointer 0x55a43b0e8000 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** DB Stats ** 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-30T13:15:58.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** Compaction Stats [default] ** 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 113.4 0.00 0.00 1 0.001 0 0 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 113.4 0.00 0.00 1 0.001 0 0 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 113.4 0.00 0.00 1 0.001 0 0 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** Compaction Stats [default] ** 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 113.4 0.00 0.00 1 0.001 0 0 2024-03-30T13:15:58.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative compaction: 0.00 GB write, 18.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval compaction: 0.00 GB write, 18.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** Compaction Stats [default] ** 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:15:58.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 113.4 0.00 0.00 1 0.001 0 0 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 113.4 0.00 0.00 1 0.001 0 0 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** Compaction Stats [default] ** 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 113.4 0.00 0.00 1 0.001 0 0 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative compaction: 0.00 GB write, 18.38 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:15:58.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: starting mon.smithi159 rank 0 at public addrs [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] at bind addrs [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi159 fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159@-1(???) e1 preinit fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159@-1(???).mds e1 new map 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159@-1(???).mds e1 print_map 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: e1 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: legacy client fscid: -1 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: No filesystems configured 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-30T13:15:58.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 bash[24768]: d58042490a7e0511e9cbe78be6c4f13b4bd32a4e5b2791c1600fa136ef0e6f3c 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 systemd[1]: Started Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mon.smithi159 is new leader, mons smithi159 in quorum (ranks 0) 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: monmap e1: 1 mons at {smithi159=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0]} 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: fsmap 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: osdmap e1: 0 total, 0 up, 0 in 2024-03-30T13:15:58.603 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:15:58 smithi159 ceph-mon[24787]: mgrmap e1: no daemons active 2024-03-30T13:15:59.543 INFO:teuthology.orchestra.run.smithi159.stderr:Wrote config to /etc/ceph/ceph.conf 2024-03-30T13:15:59.543 INFO:teuthology.orchestra.run.smithi159.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-30T13:15:59.543 INFO:teuthology.orchestra.run.smithi159.stderr:Creating mgr... 2024-03-30T13:15:59.543 INFO:teuthology.orchestra.run.smithi159.stderr:Verifying port 9283 ... 2024-03-30T13:15:59.822 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Failed to reset failed state of unit ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mgr.smithi159.fnasly.service: Unit ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mgr.smithi159.fnasly.service not loaded. 2024-03-30T13:15:59.829 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4.target.wants/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mgr.smithi159.fnasly.service → /etc/systemd/system/ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@.service. 2024-03-30T13:16:00.142 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:00 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1368237712' entity='client.admin' 2024-03-30T13:16:04.033 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: disabled 2024-03-30T13:16:04.040 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: inactive 2024-03-30T13:16:04.040 INFO:teuthology.orchestra.run.smithi159.stderr:firewalld.service is not enabled 2024-03-30T13:16:04.041 INFO:teuthology.orchestra.run.smithi159.stderr:Not possible to enable service . firewalld.service is not available 2024-03-30T13:16:04.049 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: disabled 2024-03-30T13:16:04.056 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: inactive 2024-03-30T13:16:04.057 INFO:teuthology.orchestra.run.smithi159.stderr:firewalld.service is not enabled 2024-03-30T13:16:04.057 INFO:teuthology.orchestra.run.smithi159.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-03-30T13:16:04.057 INFO:teuthology.orchestra.run.smithi159.stderr:Waiting for mgr to start... 2024-03-30T13:16:04.057 INFO:teuthology.orchestra.run.smithi159.stderr:Waiting for mgr... 2024-03-30T13:16:07.208 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:07 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2164748498' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: { 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "fsid": "9058771a-ee97-11ee-b647-cb9ed24678a4", 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "health": { 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "checks": {}, 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "mutes": [] 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:07.861 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "quorum": [ 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 0 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: ], 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "smithi159" 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: ], 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "monmap": { 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:07.862 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "osdmap": { 2024-03-30T13:16:07.863 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-30T13:16:07.863 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-30T13:16:07.863 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-30T13:16:07.863 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-30T13:16:07.863 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-30T13:16:07.866 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-30T13:16:07.867 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-30T13:16:07.867 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:07.867 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "pgmap": { 2024-03-30T13:16:07.867 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-30T13:16:07.867 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-30T13:16:07.867 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-30T13:16:07.867 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-30T13:16:07.867 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "fsmap": { 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-30T13:16:07.868 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "available": false, 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "modules": [ 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "iostat", 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "nfs", 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "restful" 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: ], 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "services": {} 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "servicemap": { 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "modified": "2024-03-30T13:15:53.971079+0000", 2024-03-30T13:16:07.869 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "services": {} 2024-03-30T13:16:07.870 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:07.870 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-30T13:16:07.870 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: } 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: Activating manager daemon smithi159.fnasly 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: mgrmap e2: smithi159.fnasly(active, starting, since 0.0027351s) 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:16:09.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:16:09.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:16:09.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:09.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:09.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:09 smithi159 ceph-mon[24787]: from='mgr.14100 172.21.15.159:0/338856351' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:11.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:10 smithi159 ceph-mon[24787]: mgrmap e3: smithi159.fnasly(active, since 1.00668s) 2024-03-30T13:16:11.101 INFO:teuthology.orchestra.run.smithi159.stderr:mgr not available, waiting (1/15)... 2024-03-30T13:16:12.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:11 smithi159 ceph-mon[24787]: mgrmap e4: smithi159.fnasly(active, since 2s) 2024-03-30T13:16:13.786 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:16:13.786 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: { 2024-03-30T13:16:13.786 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "fsid": "9058771a-ee97-11ee-b647-cb9ed24678a4", 2024-03-30T13:16:13.786 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "health": { 2024-03-30T13:16:13.786 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-30T13:16:13.786 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "checks": {}, 2024-03-30T13:16:13.786 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "mutes": [] 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "quorum": [ 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 0 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: ], 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "smithi159" 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: ], 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "quorum_age": 15, 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "monmap": { 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:13.787 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "osdmap": { 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "pgmap": { 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-30T13:16:13.788 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "fsmap": { 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:13.789 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "available": true, 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "modules": [ 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "iostat", 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "nfs", 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "restful" 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: ], 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "services": {} 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "servicemap": { 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "modified": "2024-03-30T13:15:53.971079+0000", 2024-03-30T13:16:13.790 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "services": {} 2024-03-30T13:16:13.791 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: }, 2024-03-30T13:16:13.791 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-30T13:16:13.791 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: } 2024-03-30T13:16:14.347 INFO:teuthology.orchestra.run.smithi159.stderr:mgr is available 2024-03-30T13:16:14.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:14 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/260446365' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-30T13:16:14.912 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:16:14.912 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: [global] 2024-03-30T13:16:14.912 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: fsid = 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: [mgr] 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: [osd] 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-03-30T13:16:14.913 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-03-30T13:16:15.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:15 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/509945021' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-30T13:16:15.514 INFO:teuthology.orchestra.run.smithi159.stderr:Enabling cephadm module... 2024-03-30T13:16:17.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:16 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1941793972' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-30T13:16:18.075 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: { 2024-03-30T13:16:18.075 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 5, 2024-03-30T13:16:18.075 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "available": true, 2024-03-30T13:16:18.075 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "active_name": "smithi159.fnasly", 2024-03-30T13:16:18.075 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_standby": 0 2024-03-30T13:16:18.075 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: } 2024-03-30T13:16:18.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:17 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1941793972' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-30T13:16:18.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:17 smithi159 ceph-mon[24787]: mgrmap e5: smithi159.fnasly(active, since 7s) 2024-03-30T13:16:18.674 INFO:teuthology.orchestra.run.smithi159.stderr:Waiting for the mgr to restart... 2024-03-30T13:16:18.674 INFO:teuthology.orchestra.run.smithi159.stderr:Waiting for mgr epoch 5... 2024-03-30T13:16:18.909 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:18 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3335641002' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-30T13:16:23.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:22 smithi159 ceph-mon[24787]: Active manager daemon smithi159.fnasly restarted 2024-03-30T13:16:23.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:22 smithi159 ceph-mon[24787]: Activating manager daemon smithi159.fnasly 2024-03-30T13:16:23.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:22 smithi159 ceph-mon[24787]: osdmap e2: 0 total, 0 up, 0 in 2024-03-30T13:16:23.755 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: { 2024-03-30T13:16:23.755 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-03-30T13:16:23.755 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "initialized": true 2024-03-30T13:16:23.755 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: } 2024-03-30T13:16:24.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: mgrmap e6: smithi159.fnasly(active, starting, since 0.0525352s) 2024-03-30T13:16:24.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:16:24.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:16:24.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:23 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:16:24.313 INFO:teuthology.orchestra.run.smithi159.stderr:mgr epoch 5 is available 2024-03-30T13:16:24.313 INFO:teuthology.orchestra.run.smithi159.stderr:Setting orchestrator backend to cephadm... 2024-03-30T13:16:24.894 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:24 smithi159 ceph-mon[24787]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-30T13:16:24.895 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:24 smithi159 ceph-mon[24787]: mgrmap e7: smithi159.fnasly(active, since 1.0593s) 2024-03-30T13:16:24.895 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:24 smithi159 ceph-mon[24787]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-30T13:16:24.895 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:24 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:24.895 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:24 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:16:25.935 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: value unchanged 2024-03-30T13:16:26.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:25 smithi159 ceph-mon[24787]: [30/Mar/2024:13:16:24] ENGINE Bus STARTING 2024-03-30T13:16:26.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:25 smithi159 ceph-mon[24787]: [30/Mar/2024:13:16:24] ENGINE Serving on https://172.21.15.159:7150 2024-03-30T13:16:26.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:25 smithi159 ceph-mon[24787]: [30/Mar/2024:13:16:24] ENGINE Bus STARTED 2024-03-30T13:16:26.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:25 smithi159 ceph-mon[24787]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:26.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:25 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:26.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:25 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:16:26.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:25 smithi159 ceph-mon[24787]: mgrmap e8: smithi159.fnasly(active, since 2s) 2024-03-30T13:16:26.492 INFO:teuthology.orchestra.run.smithi159.stderr:Generating ssh key... 2024-03-30T13:16:27.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:26 smithi159 ceph-mon[24787]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:28.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:28 smithi159 ceph-mon[24787]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:28.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:28 smithi159 ceph-mon[24787]: Generating ssh key... 2024-03-30T13:16:28.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:28 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:28.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:28 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:28.455 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvhxxYpGqfY8lpQsPNddV6EMXTgFzmA89KLGW2ziIA9JpjIR1n6ExFuP3mbUEnJW3f1FBNq3qmpJ1NWztjtJsVLpsGYzgGA0f4rwySPB7U2KLBs2O9SR/snKHS3JyHDGYObVx68H66Xt2gP0PTTxxL+QLi5lmXRaU+cQby7mvCSBiuLxy5gqS/6BQO2q2Y5cOclgx7NiofSRO34rZjAsEPSv58s1btZhR1f4HZGvHDXhgj9cR1UT6mmP7v1C7rZhtMxycXxzh2/Os9miHUDjhRRW1S49XDoHP+uufGN1ZoLH4OrrX7L7/74iy3SBO19wsK4RDUqLpFiZAL68oIyXOKu6Rd5C0Kruja0GoLv5a/9zjiKulPs/XHUlLGiYKE6mwH94pVrX9/B8V+mDjNRVzhaEY9d94UwuW8+lSDnUnsPOtADyxBBKLG1lj9nzEP74K/Uy6b7RR0MxsoTmy/pDgth4lJuVau98R01KkLJ2rnxVZnb+Dtlj7nqkTluSzaIX0= ceph-9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:16:28.997 INFO:teuthology.orchestra.run.smithi159.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-30T13:16:28.997 INFO:teuthology.orchestra.run.smithi159.stderr:Adding key to root@localhost authorized_keys... 2024-03-30T13:16:28.997 INFO:teuthology.orchestra.run.smithi159.stderr:Adding host smithi159... 2024-03-30T13:16:29.668 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:29 smithi159 ceph-mon[24787]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:30.611 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Added host 'smithi159' with addr '172.21.15.159' 2024-03-30T13:16:30.622 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:30 smithi159 ceph-mon[24787]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi159", "addr": "172.21.15.159", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:31.113 INFO:teuthology.orchestra.run.smithi159.stderr:Deploying mon service with default placement... 2024-03-30T13:16:31.711 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Scheduled mon update... 2024-03-30T13:16:31.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:31 smithi159 ceph-mon[24787]: Deploying cephadm binary to smithi159 2024-03-30T13:16:31.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:31 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:31.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:31 smithi159 ceph-mon[24787]: Added host smithi159 2024-03-30T13:16:31.756 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:31 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:16:32.412 INFO:teuthology.orchestra.run.smithi159.stderr:Deploying mgr service with default placement... 2024-03-30T13:16:33.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:32 smithi159 ceph-mon[24787]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:33.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:32 smithi159 ceph-mon[24787]: Saving service mon spec with placement count:5 2024-03-30T13:16:33.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:32 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:33.135 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-03-30T13:16:33.747 INFO:teuthology.orchestra.run.smithi159.stderr:Deploying crash service with default placement... 2024-03-30T13:16:34.298 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:34 smithi159 ceph-mon[24787]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:34.298 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:34 smithi159 ceph-mon[24787]: Saving service mgr spec with placement count:2 2024-03-30T13:16:34.298 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:34 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:34.347 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Scheduled crash update... 2024-03-30T13:16:35.173 INFO:teuthology.orchestra.run.smithi159.stderr:Deploying prometheus service with default placement... 2024-03-30T13:16:35.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:35 smithi159 ceph-mon[24787]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:35.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:35 smithi159 ceph-mon[24787]: Saving service crash spec with placement * 2024-03-30T13:16:35.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:35 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:35.802 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-03-30T13:16:36.481 INFO:teuthology.orchestra.run.smithi159.stderr:Deploying grafana service with default placement... 2024-03-30T13:16:36.898 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:36 smithi159 ceph-mon[24787]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:36.898 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:36 smithi159 ceph-mon[24787]: Saving service prometheus spec with placement count:1 2024-03-30T13:16:36.898 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:36 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:36.898 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:36 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:37.094 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-03-30T13:16:37.562 INFO:teuthology.orchestra.run.smithi159.stderr:Deploying node-exporter service with default placement... 2024-03-30T13:16:38.186 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-03-30T13:16:38.198 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:37 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:38.198 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:37 smithi159 ceph-mon[24787]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:38.198 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:37 smithi159 ceph-mon[24787]: Saving service grafana spec with placement count:1 2024-03-30T13:16:38.198 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:37 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:38.762 INFO:teuthology.orchestra.run.smithi159.stderr:Deploying alertmanager service with default placement... 2024-03-30T13:16:39.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:39 smithi159 ceph-mon[24787]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:39.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:39 smithi159 ceph-mon[24787]: Saving service node-exporter spec with placement * 2024-03-30T13:16:39.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:39 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:39.801 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-03-30T13:16:40.369 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:40 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:40.369 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:40 smithi159 ceph-mon[24787]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:40.369 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:40 smithi159 ceph-mon[24787]: Saving service alertmanager spec with placement count:1 2024-03-30T13:16:40.369 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:40 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:40.369 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:40 smithi159 ceph-mon[24787]: from='mgr.14116 172.21.15.159:0/2794527889' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:42.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:41 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2690201469' entity='client.admin' 2024-03-30T13:16:43.598 INFO:teuthology.orchestra.run.smithi159.stderr:Enabling the dashboard module... 2024-03-30T13:16:43.932 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:43 smithi159 ceph-mon[24787]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:16:43.932 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:43 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2801373256' entity='client.admin' 2024-03-30T13:16:45.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:44 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1638789570' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-30T13:16:46.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:45 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1638789570' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-30T13:16:46.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:45 smithi159 ceph-mon[24787]: mgrmap e9: smithi159.fnasly(active, since 22s) 2024-03-30T13:16:46.152 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: { 2024-03-30T13:16:46.152 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "epoch": 9, 2024-03-30T13:16:46.152 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "available": true, 2024-03-30T13:16:46.152 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "active_name": "smithi159.fnasly", 2024-03-30T13:16:46.152 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "num_standby": 0 2024-03-30T13:16:46.152 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: } 2024-03-30T13:16:46.678 INFO:teuthology.orchestra.run.smithi159.stderr:Waiting for the mgr to restart... 2024-03-30T13:16:46.679 INFO:teuthology.orchestra.run.smithi159.stderr:Waiting for mgr epoch 9... 2024-03-30T13:16:47.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:46 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1785753953' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-30T13:16:51.618 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: Active manager daemon smithi159.fnasly restarted 2024-03-30T13:16:51.618 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: Activating manager daemon smithi159.fnasly 2024-03-30T13:16:51.618 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: osdmap e3: 0 total, 0 up, 0 in 2024-03-30T13:16:51.618 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: mgrmap e10: smithi159.fnasly(active, starting, since 0.0553327s) 2024-03-30T13:16:51.618 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:16:51.618 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:16:51.618 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: [30/Mar/2024:13:16:50] ENGINE Bus STARTING 2024-03-30T13:16:51.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:51 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:51.620 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: { 2024-03-30T13:16:51.620 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-03-30T13:16:51.620 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: "initialized": true 2024-03-30T13:16:51.620 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: } 2024-03-30T13:16:52.272 INFO:teuthology.orchestra.run.smithi159.stderr:mgr epoch 9 is available 2024-03-30T13:16:52.272 INFO:teuthology.orchestra.run.smithi159.stderr:Generating a dashboard self-signed certificate... 2024-03-30T13:16:52.824 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:52 smithi159 ceph-mon[24787]: [30/Mar/2024:13:16:50] ENGINE Serving on https://172.21.15.159:7150 2024-03-30T13:16:52.824 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:52 smithi159 ceph-mon[24787]: [30/Mar/2024:13:16:50] ENGINE Bus STARTED 2024-03-30T13:16:52.824 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:52 smithi159 ceph-mon[24787]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-30T13:16:52.824 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:52 smithi159 ceph-mon[24787]: mgrmap e11: smithi159.fnasly(active, since 1.05777s) 2024-03-30T13:16:52.825 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:52 smithi159 ceph-mon[24787]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-30T13:16:52.825 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:52 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:52.989 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: Self-signed certificate created 2024-03-30T13:16:53.528 INFO:teuthology.orchestra.run.smithi159.stderr:Creating initial admin user... 2024-03-30T13:16:54.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:53 smithi159 ceph-mon[24787]: from='client.14170 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:54.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:53 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:54.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:53 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:54.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:53 smithi159 ceph-mon[24787]: mgrmap e12: smithi159.fnasly(active, since 2s) 2024-03-30T13:16:54.417 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$uKFH32fAgpwcHKFJq5c4J.Z0mqOel2Y2RPGfTASMWxTVxXkCfRZxW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1711804614, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-30T13:16:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:55 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:55 smithi159 ceph-mon[24787]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:16:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:55 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:16:55.749 INFO:teuthology.orchestra.run.smithi159.stderr:Fetching dashboard port number... 2024-03-30T13:16:56.689 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: 8443 2024-03-30T13:16:57.385 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: disabled 2024-03-30T13:16:57.392 INFO:teuthology.orchestra.run.smithi159.stderr:systemctl: inactive 2024-03-30T13:16:57.393 INFO:teuthology.orchestra.run.smithi159.stderr:firewalld.service is not enabled 2024-03-30T13:16:57.393 INFO:teuthology.orchestra.run.smithi159.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-30T13:16:57.395 INFO:teuthology.orchestra.run.smithi159.stderr:Ceph Dashboard is now available at: 2024-03-30T13:16:57.396 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:16:57.396 INFO:teuthology.orchestra.run.smithi159.stderr: URL: https://smithi159.front.sepia.ceph.com:8443/ 2024-03-30T13:16:57.396 INFO:teuthology.orchestra.run.smithi159.stderr: User: admin 2024-03-30T13:16:57.396 INFO:teuthology.orchestra.run.smithi159.stderr: Password: 1uw765zh5t 2024-03-30T13:16:57.396 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:16:57.396 INFO:teuthology.orchestra.run.smithi159.stderr:Enabling autotune for osd_memory_target 2024-03-30T13:16:57.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:57 smithi159 ceph-mon[24787]: mgrmap e13: smithi159.fnasly(active, since 6s) 2024-03-30T13:16:57.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:16:57 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/4148596414' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-30T13:16:59.427 INFO:teuthology.orchestra.run.smithi159.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-03-30T13:16:59.973 INFO:teuthology.orchestra.run.smithi159.stderr:You can access the Ceph CLI with: 2024-03-30T13:16:59.973 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:16:59.973 INFO:teuthology.orchestra.run.smithi159.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-30T13:16:59.973 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:16:59.973 INFO:teuthology.orchestra.run.smithi159.stderr:Please consider enabling telemetry to help improve Ceph: 2024-03-30T13:16:59.973 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:16:59.974 INFO:teuthology.orchestra.run.smithi159.stderr: ceph telemetry on 2024-03-30T13:16:59.974 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:16:59.974 INFO:teuthology.orchestra.run.smithi159.stderr:For more information see: 2024-03-30T13:16:59.974 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:16:59.974 INFO:teuthology.orchestra.run.smithi159.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-03-30T13:16:59.974 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:16:59.974 INFO:teuthology.orchestra.run.smithi159.stderr:Bootstrap complete. 2024-03-30T13:17:00.010 INFO:tasks.cephadm:Fetching config... 2024-03-30T13:17:00.011 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:17:00.011 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-30T13:17:00.027 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-30T13:17:00.028 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:17:00.028 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-30T13:17:00.083 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-30T13:17:00.083 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:17:00.083 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/keyring of=/dev/stdout 2024-03-30T13:17:00.150 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-30T13:17:00.151 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:17:00.151 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-30T13:17:00.205 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-30T13:17:00.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvhxxYpGqfY8lpQsPNddV6EMXTgFzmA89KLGW2ziIA9JpjIR1n6ExFuP3mbUEnJW3f1FBNq3qmpJ1NWztjtJsVLpsGYzgGA0f4rwySPB7U2KLBs2O9SR/snKHS3JyHDGYObVx68H66Xt2gP0PTTxxL+QLi5lmXRaU+cQby7mvCSBiuLxy5gqS/6BQO2q2Y5cOclgx7NiofSRO34rZjAsEPSv58s1btZhR1f4HZGvHDXhgj9cR1UT6mmP7v1C7rZhtMxycXxzh2/Os9miHUDjhRRW1S49XDoHP+uufGN1ZoLH4OrrX7L7/74iy3SBO19wsK4RDUqLpFiZAL68oIyXOKu6Rd5C0Kruja0GoLv5a/9zjiKulPs/XHUlLGiYKE6mwH94pVrX9/B8V+mDjNRVzhaEY9d94UwuW8+lSDnUnsPOtADyxBBKLG1lj9nzEP74K/Uy6b7RR0MxsoTmy/pDgth4lJuVau98R01KkLJ2rnxVZnb+Dtlj7nqkTluSzaIX0= ceph-9058771a-ee97-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-30T13:17:00.288 INFO:teuthology.orchestra.run.smithi159.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvhxxYpGqfY8lpQsPNddV6EMXTgFzmA89KLGW2ziIA9JpjIR1n6ExFuP3mbUEnJW3f1FBNq3qmpJ1NWztjtJsVLpsGYzgGA0f4rwySPB7U2KLBs2O9SR/snKHS3JyHDGYObVx68H66Xt2gP0PTTxxL+QLi5lmXRaU+cQby7mvCSBiuLxy5gqS/6BQO2q2Y5cOclgx7NiofSRO34rZjAsEPSv58s1btZhR1f4HZGvHDXhgj9cR1UT6mmP7v1C7rZhtMxycXxzh2/Os9miHUDjhRRW1S49XDoHP+uufGN1ZoLH4OrrX7L7/74iy3SBO19wsK4RDUqLpFiZAL68oIyXOKu6Rd5C0Kruja0GoLv5a/9zjiKulPs/XHUlLGiYKE6mwH94pVrX9/B8V+mDjNRVzhaEY9d94UwuW8+lSDnUnsPOtADyxBBKLG1lj9nzEP74K/Uy6b7RR0MxsoTmy/pDgth4lJuVau98R01KkLJ2rnxVZnb+Dtlj7nqkTluSzaIX0= ceph-9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:17:00.302 DEBUG:teuthology.orchestra.run.smithi187:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvhxxYpGqfY8lpQsPNddV6EMXTgFzmA89KLGW2ziIA9JpjIR1n6ExFuP3mbUEnJW3f1FBNq3qmpJ1NWztjtJsVLpsGYzgGA0f4rwySPB7U2KLBs2O9SR/snKHS3JyHDGYObVx68H66Xt2gP0PTTxxL+QLi5lmXRaU+cQby7mvCSBiuLxy5gqS/6BQO2q2Y5cOclgx7NiofSRO34rZjAsEPSv58s1btZhR1f4HZGvHDXhgj9cR1UT6mmP7v1C7rZhtMxycXxzh2/Os9miHUDjhRRW1S49XDoHP+uufGN1ZoLH4OrrX7L7/74iy3SBO19wsK4RDUqLpFiZAL68oIyXOKu6Rd5C0Kruja0GoLv5a/9zjiKulPs/XHUlLGiYKE6mwH94pVrX9/B8V+mDjNRVzhaEY9d94UwuW8+lSDnUnsPOtADyxBBKLG1lj9nzEP74K/Uy6b7RR0MxsoTmy/pDgth4lJuVau98R01KkLJ2rnxVZnb+Dtlj7nqkTluSzaIX0= ceph-9058771a-ee97-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-30T13:17:00.348 INFO:teuthology.orchestra.run.smithi187.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvhxxYpGqfY8lpQsPNddV6EMXTgFzmA89KLGW2ziIA9JpjIR1n6ExFuP3mbUEnJW3f1FBNq3qmpJ1NWztjtJsVLpsGYzgGA0f4rwySPB7U2KLBs2O9SR/snKHS3JyHDGYObVx68H66Xt2gP0PTTxxL+QLi5lmXRaU+cQby7mvCSBiuLxy5gqS/6BQO2q2Y5cOclgx7NiofSRO34rZjAsEPSv58s1btZhR1f4HZGvHDXhgj9cR1UT6mmP7v1C7rZhtMxycXxzh2/Os9miHUDjhRRW1S49XDoHP+uufGN1ZoLH4OrrX7L7/74iy3SBO19wsK4RDUqLpFiZAL68oIyXOKu6Rd5C0Kruja0GoLv5a/9zjiKulPs/XHUlLGiYKE6mwH94pVrX9/B8V+mDjNRVzhaEY9d94UwuW8+lSDnUnsPOtADyxBBKLG1lj9nzEP74K/Uy6b7RR0MxsoTmy/pDgth4lJuVau98R01KkLJ2rnxVZnb+Dtlj7nqkTluSzaIX0= ceph-9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:17:00.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-30T13:17:00.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:00 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/4217620134' entity='client.admin' 2024-03-30T13:17:03.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:02 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:03.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:02 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd/host:smithi159", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:17:03.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:02 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:03.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:02 smithi159 ceph-mon[24787]: Deploying daemon alertmanager.smithi159 on smithi159 2024-03-30T13:17:03.800 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-30T13:17:03.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-30T13:17:04.424 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:04 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3122378351' entity='client.admin' 2024-03-30T13:17:07.137 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi187 2024-03-30T13:17:07.137 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-30T13:17:07.137 DEBUG:teuthology.orchestra.run.smithi187:> dd of=/etc/ceph/ceph.conf 2024-03-30T13:17:07.156 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-30T13:17:07.156 DEBUG:teuthology.orchestra.run.smithi187:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:17:07.214 INFO:tasks.cephadm:Adding host smithi187 to orchestrator... 2024-03-30T13:17:07.214 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch host add smithi187 2024-03-30T13:17:07.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:07 smithi159 ceph-mon[24787]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:17:07.534 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:07 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:11.245 INFO:teuthology.orchestra.run.smithi159.stdout:Added host 'smithi187' with addr '172.21.15.187' 2024-03-30T13:17:11.820 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:11 smithi159 ceph-mon[24787]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi187", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:17:11.820 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:11 smithi159 ceph-mon[24787]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:11.820 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:11 smithi159 ceph-mon[24787]: Deploying cephadm binary to smithi187 2024-03-30T13:17:11.820 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:11 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:12.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch host ls --format=json 2024-03-30T13:17:12.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:12 smithi159 ceph-mon[24787]: Added host smithi187 2024-03-30T13:17:14.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:13 smithi159 ceph-mon[24787]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:14.856 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:17:14.856 INFO:teuthology.orchestra.run.smithi159.stdout:[{"addr": "172.21.15.159", "hostname": "smithi159", "labels": [], "status": ""}, {"addr": "172.21.15.187", "hostname": "smithi187", "labels": [], "status": ""}] 2024-03-30T13:17:15.842 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-30T13:17:15.842 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd crush tunables default 2024-03-30T13:17:16.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:15 smithi159 ceph-mon[24787]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:16.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:15 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:16.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:15 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi159", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:17:16.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:15 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi159", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-30T13:17:16.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:15 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:17:16.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:15 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:16.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:16 smithi159 ceph-mon[24787]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:17:16.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:16 smithi159 ceph-mon[24787]: Deploying daemon crash.smithi159 on smithi159 2024-03-30T13:17:17.966 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:17 smithi159 ceph-mon[24787]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:19.425 INFO:teuthology.orchestra.run.smithi159.stderr:adjusted tunables profile to default 2024-03-30T13:17:19.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:19 smithi159 ceph-mon[24787]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:19.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:19 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2548086567' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-30T13:17:19.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:19 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:19.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:19 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:19.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:19 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:19.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:19 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-30T13:17:19.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:19 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:20.074 INFO:tasks.cephadm:Adding mon.smithi159 on smithi159 2024-03-30T13:17:20.074 INFO:tasks.cephadm:Adding mon.smithi187 on smithi187 2024-03-30T13:17:20.074 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch apply mon '2;smithi159:172.21.15.159=smithi159;smithi187:172.21.15.187=smithi187' 2024-03-30T13:17:20.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:20 smithi159 ceph-mon[24787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-30T13:17:20.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:20 smithi159 ceph-mon[24787]: Deploying daemon grafana.smithi159 on smithi159 2024-03-30T13:17:20.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:20 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2548086567' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-30T13:17:20.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:20 smithi159 ceph-mon[24787]: osdmap e4: 0 total, 0 up, 0 in 2024-03-30T13:17:21.874 INFO:teuthology.orchestra.run.smithi187.stdout:Scheduled mon update... 2024-03-30T13:17:22.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:21 smithi159 ceph-mon[24787]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:22.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:21 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:22.319 DEBUG:teuthology.orchestra.run.smithi187:mon.smithi187> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi187.service 2024-03-30T13:17:22.321 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:22.322 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:23.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:22 smithi159 ceph-mon[24787]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi159:172.21.15.159=smithi159;smithi187:172.21.15.187=smithi187", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:17:23.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:22 smithi159 ceph-mon[24787]: Saving service mon spec with placement smithi159:172.21.15.159=smithi159;smithi187:172.21.15.187=smithi187;count:2 2024-03-30T13:17:23.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:22 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:23.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:22 smithi159 ceph-mon[24787]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:24.086 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:24.086 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:24.088 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:24.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:24 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/3148359481' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:25.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:25 smithi159 ceph-mon[24787]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:25.655 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:25.655 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:27.399 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:27.399 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:27.401 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:28.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:27 smithi159 ceph-mon[24787]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:28.030 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:27 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/1458393101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:28.915 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:28.916 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:29.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:29 smithi159 ceph-mon[24787]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:30.727 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:30.727 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:30.729 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:32.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:31 smithi159 ceph-mon[24787]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:32.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:31 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/3571968792' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:32.242 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:32.243 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:34.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:33 smithi159 ceph-mon[24787]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:34.086 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:34.086 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:34.088 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:35.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:34 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/3201948288' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:35.694 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:35.695 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:35.987 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:35 smithi159 ceph-mon[24787]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:37.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:37 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:37.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:37 smithi159 ceph-mon[24787]: Deploying daemon node-exporter.smithi159 on smithi159 2024-03-30T13:17:37.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:37 smithi159 ceph-mon[24787]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:37.533 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:37.533 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:37.535 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:38.542 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:38 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/2198305315' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:39.192 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:39.192 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:39.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:39 smithi159 ceph-mon[24787]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:17:39.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:39 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:39.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:39 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-30T13:17:41.077 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:41.077 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:41.079 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:41.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:41 smithi159 ceph-mon[24787]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-30T13:17:41.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:41 smithi159 ceph-mon[24787]: mgrmap e14: smithi159.fnasly(active, since 49s) 2024-03-30T13:17:41.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:41 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/3248137131' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:42.610 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:42.611 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:44.283 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:44.283 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:44.285 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:44.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:44 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/366634811' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:45.797 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:45.797 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:46.404 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:46 smithi159 ceph-mon[24787]: Active manager daemon smithi159.fnasly restarted 2024-03-30T13:17:46.404 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:46 smithi159 ceph-mon[24787]: Activating manager daemon smithi159.fnasly 2024-03-30T13:17:46.404 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:46 smithi159 ceph-mon[24787]: osdmap e5: 0 total, 0 up, 0 in 2024-03-30T13:17:47.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: mgrmap e15: smithi159.fnasly(active, starting, since 0.0533351s) 2024-03-30T13:17:47.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:17:47.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:17:47.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:17:47.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:17:47.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:17:47.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:17:47.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:17:47.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:17:47.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:17:47.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:17:47.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:47.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:17:47.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:17:47.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:47.699 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:47.699 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:47.701 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:48.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:48 smithi159 ceph-mon[24787]: [30/Mar/2024:13:17:46] ENGINE Bus STARTING 2024-03-30T13:17:48.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:48 smithi159 ceph-mon[24787]: [30/Mar/2024:13:17:47] ENGINE Serving on https://172.21.15.159:7150 2024-03-30T13:17:48.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:48 smithi159 ceph-mon[24787]: [30/Mar/2024:13:17:47] ENGINE Bus STARTED 2024-03-30T13:17:48.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:48 smithi159 ceph-mon[24787]: mgrmap e16: smithi159.fnasly(active, since 1.06145s) 2024-03-30T13:17:48.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:48 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:48.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:48 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/4238556475' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:49.317 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:49.318 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:49.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:49.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:49 smithi159 ceph-mon[24787]: mgrmap e17: smithi159.fnasly(active, since 2s) 2024-03-30T13:17:50.385 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:50 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:50.385 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:50 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:51.020 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:51.020 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:51.021 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:51.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:51.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd/host:smithi159", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:17:51.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:51 smithi159 ceph-mon[24787]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:17:51.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:51 smithi159 ceph-mon[24787]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:17:51.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:51.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:51 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/1629565826' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:53.239 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:53.239 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd/host:smithi187", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:17:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:17:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-30T13:17:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:17:55.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:54 smithi159 ceph-mon[24787]: Deploying daemon crash.smithi187 on smithi187 2024-03-30T13:17:55.530 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:17:55.530 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:17:55.532 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:17:56.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:55 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/1880011142' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:17:57.438 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:17:57.438 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:17:59.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:17:59.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:17:59.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-30T13:17:59.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:17:59.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:17:59.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:17:59 smithi159 ceph-mon[24787]: Deploying daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:18:00.884 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:18:00.885 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:18:00.886 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:18:01.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:00 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/1935656822' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:02.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:02.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:02.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:18:02.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:02.763 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:18:02.763 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:18:03.406 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:03 smithi159 ceph-mon[24787]: Deploying daemon mon.smithi187 on smithi187 2024-03-30T13:18:04.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:04 smithi187 systemd[1]: Starting Ceph mon.smithi187 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:18:05.369 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:18:05.370 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:15:51.155583Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-30T13:18:05.372 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-30T13:18:05.517 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 podman[27464]: 2024-03-30 13:18:05.195018661 +0000 UTC m=+0.026767944 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-03-30T13:18:05.517 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 podman[27464]: 2024-03-30 13:18:05.304418977 +0000 UTC m=+0.136168254 container create 9f317cdceccec009300e17c0ad4860a3c36758b0f37a06ff5a2a883b003d0e3d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, GIT_CLEAN=True, io.openshift.expose-services=, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, name=centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, version=8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc.) 2024-03-30T13:18:05.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:05 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/2731519357' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 podman[27464]: 2024-03-30 13:18:05.767145834 +0000 UTC m=+0.598895130 container init 9f317cdceccec009300e17c0ad4860a3c36758b0f37a06ff5a2a883b003d0e3d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vendor=Red Hat, Inc., version=8, io.openshift.tags=base centos centos-stream, release=754, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, vcs-type=git, ceph=True, GIT_CLEAN=True, architecture=x86_64, io.openshift.expose-services=, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-03-30T13:18:06.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 podman[27464]: 2024-03-30 13:18:05.770483769 +0000 UTC m=+0.602233049 container start 9f317cdceccec009300e17c0ad4860a3c36758b0f37a06ff5a2a883b003d0e3d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.expose-services=, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8) 2024-03-30T13:18:06.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: set uid:gid to 167:167 (ceph:ceph) 2024-03-30T13:18:06.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-03-30T13:18:06.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pidfile_write: ignore empty --pid-file 2024-03-30T13:18:06.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: load: jerasure load: lrc 2024-03-30T13:18:06.025 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: RocksDB version: 6.15.5 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Compile date Apr 18 2022 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: DB SUMMARY 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: DB Session ID: JK409FJCUQ8DK53PCKDU 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: CURRENT file: CURRENT 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: IDENTITY file: IDENTITY 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi187/store.db dir, Total Num: 0, files: 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi187/store.db: 000004.log size: 511 ; 2024-03-30T13:18:06.026 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.error_if_exists: 0 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.create_if_missing: 0 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.paranoid_checks: 1 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.env: 0x562e9fac6860 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.fs: Posix File System 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.info_log: 0x562ea0b4ffa0 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_file_opening_threads: 16 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.statistics: (nil) 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.use_fsync: 0 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_log_file_size: 0 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-30T13:18:06.027 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.keep_log_file_num: 1000 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.recycle_log_file_num: 0 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.allow_fallocate: 1 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.allow_mmap_reads: 0 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.allow_mmap_writes: 0 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.use_direct_reads: 0 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.create_missing_column_families: 0 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.db_log_dir: 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi187/store.db 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-30T13:18:06.028 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.advise_random_on_open: 1 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.db_write_buffer_size: 0 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.write_buffer_manager: 0x562ea0c402d0 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.rate_limiter: (nil) 2024-03-30T13:18:06.029 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.wal_recovery_mode: 2 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.enable_thread_tracking: 0 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.enable_pipelined_write: 0 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.unordered_write: 0 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.row_cache: None 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.wal_filter: None 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-30T13:18:06.030 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.allow_ingest_behind: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.preserve_deletes: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.two_write_queues: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.manual_wal_flush: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.atomic_flush: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.log_readahead_size: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.best_efforts_recovery: 0 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-30T13:18:06.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.allow_data_in_errors: 0 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.db_host_id: __hostname__ 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_background_jobs: 2 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_background_compactions: -1 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_subcompactions: 1 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_total_wal_size: 0 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-30T13:18:06.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_open_files: -1 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bytes_per_sync: 0 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_readahead_size: 0 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_background_flushes: -1 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Compression algorithms supported: 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: kZSTD supported: 0 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: kXpressCompression supported: 0 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: kLZ4HCCompression supported: 1 2024-03-30T13:18:06.033 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: kLZ4Compression supported: 1 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: kBZip2Compression supported: 0 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: kZlibCompression supported: 1 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: kSnappyCompression supported: 1 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi187/store.db/MANIFEST-000003 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.merge_operator: 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_filter: None 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_filter_factory: None 2024-03-30T13:18:06.034 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.sst_partitioner_factory: None 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562ea0b1bd00) 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: cache_index_and_filter_blocks: 1 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: pin_top_level_index_and_filter: 1 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: index_type: 0 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: data_block_index_type: 0 2024-03-30T13:18:06.035 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: index_shortening: 1 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: hash_index_allow_collision: 1 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: checksum: 1 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: no_block_cache: 0 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_cache: 0x562ea0b86170 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_cache_name: BinnedLRUCache 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_cache_options: 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: capacity : 536870912 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: num_shard_bits : 4 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: strict_capacity_limit : 0 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: high_pri_pool_ratio: 0.000 2024-03-30T13:18:06.036 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_cache_compressed: (nil) 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: persistent_cache: (nil) 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_size: 4096 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_size_deviation: 10 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_restart_interval: 16 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: index_block_restart_interval: 1 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: metadata_block_size: 4096 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: partition_filters: 0 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: use_delta_encoding: 1 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: whole_key_filtering: 1 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: verify_compression: 0 2024-03-30T13:18:06.037 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: read_amp_bytes_per_bit: 0 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: format_version: 4 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: enable_index_compression: 1 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_align: 0 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.write_buffer_size: 33554432 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_write_buffer_number: 2 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compression: NoCompression 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bottommost_compression: Disabled 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.prefix_extractor: nullptr 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.num_levels: 7 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-30T13:18:06.038 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compression_opts.level: 32767 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compression_opts.strategy: 0 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-30T13:18:06.039 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compression_opts.enabled: false 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.target_file_size_base: 67108864 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-30T13:18:06.040 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.arena_block_size: 4194304 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-30T13:18:06.041 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.disable_auto_compactions: 0 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-30T13:18:06.042 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.table_properties_collectors: 2024-03-30T13:18:06.043 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.inplace_update_support: 0 2024-03-30T13:18:06.043 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-30T13:18:06.043 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-30T13:18:06.043 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.bloom_locality: 0 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.max_successive_merges: 0 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.paranoid_file_checks: 0 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.force_consistency_checks: 1 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.report_bg_io_stats: 0 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.ttl: 2592000 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.enable_blob_files: false 2024-03-30T13:18:06.045 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.min_blob_size: 0 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.blob_file_size: 268435456 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi187/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711804685795945, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711804685796249, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711804685, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b0c0e1bd-c0df-4181-8470-ceba69edad35", "db_session_id": "JK409FJCUQ8DK53PCKDU"}} 2024-03-30T13:18:06.046 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711804685796560, "job": 1, "event": "recovery_finished"} 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi187/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x562ea0b6ca80 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: DB pointer 0x562ea0b7c000 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** DB Stats ** 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-30T13:18:06.047 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** Compaction Stats [default] ** 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** Compaction Stats [default] ** 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-30T13:18:06.048 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:18:06.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** Compaction Stats [default] ** 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** Compaction Stats [default] ** 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:18:06.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-30T13:18:06.051 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187 does not exist in monmap, will attempt to join an existing cluster 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: using public_addr v2:172.21.15.187:0/0 -> [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: starting mon.smithi187 rank -1 at public addrs [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] at bind addrs [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi187 fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(???) e0 preinit fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).mds e1 new map 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).mds e1 print_map 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: e1 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: legacy client fscid: -1 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: No filesystems configured 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-30T13:18:06.052 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/2548086567' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Deploying daemon grafana.smithi159 on smithi159 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/2548086567' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: osdmap e4: 0 total, 0 up, 0 in 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.054 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi159:172.21.15.159=smithi159;smithi187:172.21.15.187=smithi187", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Saving service mon spec with placement smithi159:172.21.15.159=smithi159;smithi187:172.21.15.187=smithi187;count:2 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/3148359481' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/1458393101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/3571968792' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.055 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/3201948288' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Deploying daemon node-exporter.smithi159 on smithi159 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/2198305315' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14158 172.21.15.159:0/3494355684' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mgrmap e14: smithi159.fnasly(active, since 49s) 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/3248137131' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/366634811' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.056 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Active manager daemon smithi159.fnasly restarted 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Activating manager daemon smithi159.fnasly 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: osdmap e5: 0 total, 0 up, 0 in 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mgrmap e15: smithi159.fnasly(active, starting, since 0.0533351s) 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:18:06.057 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: [30/Mar/2024:13:17:46] ENGINE Bus STARTING 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: [30/Mar/2024:13:17:47] ENGINE Serving on https://172.21.15.159:7150 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: [30/Mar/2024:13:17:47] ENGINE Bus STARTED 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mgrmap e16: smithi159.fnasly(active, since 1.06145s) 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/4238556475' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.058 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mgrmap e17: smithi159.fnasly(active, since 2s) 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd/host:smithi159", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/1629565826' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.059 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd/host:smithi187", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Deploying daemon crash.smithi187 on smithi187 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/1880011142' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:18:06.060 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Deploying daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/1935656822' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: Deploying daemon mon.smithi187 on smithi187 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/2731519357' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 ceph-mon[27499]: mon.smithi187@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 bash[27464]: 9f317cdceccec009300e17c0ad4860a3c36758b0f37a06ff5a2a883b003d0e3d 2024-03-30T13:18:06.061 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:05 smithi187 systemd[1]: Started Ceph mon.smithi187 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:18:07.381 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-30T13:18:07.381 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: mon.smithi159 calling monitor election 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3932788549' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: mon.smithi187 calling monitor election 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.136 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: mon.smithi159 is new leader, mons smithi159,smithi187 in quorum (ranks 0,1) 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: monmap e2: 2 mons at {smithi159=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],smithi187=[v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0]} 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: fsmap 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: osdmap e5: 0 total, 0 up, 0 in 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: mgrmap e17: smithi159.fnasly(active, since 24s) 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3932788549' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: overall HEALTH_OK 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3932788549' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3932788549' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:18:11.137 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:11.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:18:11.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: mon.smithi159 calling monitor election 2024-03-30T13:18:11.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:11.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3932788549' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: mon.smithi187 calling monitor election 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: mon.smithi159 is new leader, mons smithi159,smithi187 in quorum (ranks 0,1) 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: monmap e2: 2 mons at {smithi159=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],smithi187=[v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0]} 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: fsmap 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: osdmap e5: 0 total, 0 up, 0 in 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: mgrmap e17: smithi159.fnasly(active, since 24s) 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:18:11.256 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3932788549' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:18:11.257 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: overall HEALTH_OK 2024-03-30T13:18:11.257 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3932788549' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:18:11.257 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:11.257 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3932788549' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:18:11.257 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:12.146 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:11 smithi187 ceph-mon[27499]: Deploying daemon node-exporter.smithi187 on smithi187 2024-03-30T13:18:12.146 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:11 smithi187 ceph-mon[27499]: mgrmap e18: smithi159.fnasly(active, since 24s), standbys: smithi187.jpkaud 2024-03-30T13:18:12.146 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:11 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi187.jpkaud", "id": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:18:12.146 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:11 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:12.147 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-30T13:18:12.147 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":2,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","modified":"2024-03-30T13:18:05.820122Z","created":"2024-03-30T13:15:51.155583Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi159","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi187","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:3300","nonce":0},{"type":"v1","addr":"172.21.15.187:6789","nonce":0}]},"addr":"172.21.15.187:6789/0","public_addr":"172.21.15.187:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-30T13:18:12.147 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 2 2024-03-30T13:18:12.258 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:11 smithi159 ceph-mon[24787]: Deploying daemon node-exporter.smithi187 on smithi187 2024-03-30T13:18:12.258 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:11 smithi159 ceph-mon[24787]: mgrmap e18: smithi159.fnasly(active, since 24s), standbys: smithi187.jpkaud 2024-03-30T13:18:12.259 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:11 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi187.jpkaud", "id": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:18:12.259 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:11 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:18:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:12 smithi187 ceph-mon[27499]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:12 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/1505130132' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:13.234 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-30T13:18:13.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-03-30T13:18:13.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:12 smithi159 ceph-mon[24787]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:13.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:12 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/1505130132' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-30T13:18:14.689 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:14 smithi159 ceph-mon[24787]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:14.690 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:14 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:14.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:14 smithi187 ceph-mon[27499]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:14.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:14 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:15.081 INFO:teuthology.orchestra.run.smithi159.stdout:# minimal ceph.conf for 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:18:15.082 INFO:teuthology.orchestra.run.smithi159.stdout:[global] 2024-03-30T13:18:15.082 INFO:teuthology.orchestra.run.smithi159.stdout: fsid = 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:18:15.082 INFO:teuthology.orchestra.run.smithi159.stdout: mon_host = [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] 2024-03-30T13:18:15.605 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:15 smithi159 ceph-mon[24787]: Deploying daemon prometheus.smithi159 on smithi159 2024-03-30T13:18:15.605 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:15 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2067070610' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:15.771 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-30T13:18:15.771 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:18:15.771 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/ceph/ceph.conf 2024-03-30T13:18:15.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:15 smithi187 ceph-mon[27499]: Deploying daemon prometheus.smithi159 on smithi159 2024-03-30T13:18:15.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:15 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/2067070610' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:15.842 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:18:15.842 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:18:15.909 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-30T13:18:15.909 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/etc/ceph/ceph.conf 2024-03-30T13:18:15.944 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-30T13:18:15.944 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:18:16.014 INFO:tasks.cephadm:Deploying OSDs... 2024-03-30T13:18:16.014 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:18:16.015 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/scratch_devs of=/dev/stdout 2024-03-30T13:18:16.030 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-30T13:18:16.030 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_1 2024-03-30T13:18:16.085 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-30T13:18:16.085 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-30T13:18:16.085 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-03-30T13:18:16.085 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-30T13:18:16.085 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2024-03-30T13:18:16.085 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2024-03-30 13:17:36.899790499 +0000 2024-03-30T13:18:16.085 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2024-03-30 13:15:00.386151298 +0000 2024-03-30T13:18:16.086 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2024-03-30 13:15:00.386151298 +0000 2024-03-30T13:18:16.086 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: 2024-03-30 13:15:00.386151298 +0000 2024-03-30T13:18:16.086 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-30T13:18:16.152 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2024-03-30T13:18:16.153 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2024-03-30T13:18:16.153 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000254389 s, 2.0 MB/s 2024-03-30T13:18:16.154 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-30T13:18:16.211 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_2 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2024-03-30 13:17:36.900790480 +0000 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2024-03-30 13:15:00.721144827 +0000 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2024-03-30 13:15:00.721144827 +0000 2024-03-30T13:18:16.267 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: 2024-03-30 13:15:00.721144827 +0000 2024-03-30T13:18:16.268 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-30T13:18:16.333 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2024-03-30T13:18:16.333 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2024-03-30T13:18:16.333 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000197476 s, 2.6 MB/s 2024-03-30T13:18:16.335 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-30T13:18:16.390 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_3 2024-03-30T13:18:16.447 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-30T13:18:16.447 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-30T13:18:16.447 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-03-30T13:18:16.448 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-30T13:18:16.448 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2024-03-30T13:18:16.448 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2024-03-30 13:17:36.900790480 +0000 2024-03-30T13:18:16.448 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2024-03-30 13:15:01.051138452 +0000 2024-03-30T13:18:16.448 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2024-03-30 13:15:01.051138452 +0000 2024-03-30T13:18:16.448 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: 2024-03-30 13:15:01.051138452 +0000 2024-03-30T13:18:16.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-30T13:18:16.514 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2024-03-30T13:18:16.515 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2024-03-30T13:18:16.515 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.00019283 s, 2.7 MB/s 2024-03-30T13:18:16.516 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-30T13:18:16.574 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_4 2024-03-30T13:18:16.630 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-30T13:18:16.630 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-30T13:18:16.630 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-03-30T13:18:16.630 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-30T13:18:16.631 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2024-03-30T13:18:16.631 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2024-03-30 13:17:36.900790480 +0000 2024-03-30T13:18:16.631 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2024-03-30 13:15:01.392131865 +0000 2024-03-30T13:18:16.631 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2024-03-30 13:15:01.392131865 +0000 2024-03-30T13:18:16.631 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: 2024-03-30 13:15:01.392131865 +0000 2024-03-30T13:18:16.631 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-30T13:18:16.696 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2024-03-30T13:18:16.697 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2024-03-30T13:18:16.697 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000202735 s, 2.5 MB/s 2024-03-30T13:18:16.698 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-30T13:18:16.754 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-30T13:18:16.754 DEBUG:teuthology.orchestra.run.smithi187:> dd if=/scratch_devs of=/dev/stdout 2024-03-30T13:18:16.773 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-30T13:18:16.773 DEBUG:teuthology.orchestra.run.smithi187:> stat /dev/vg_nvme/lv_1 2024-03-30T13:18:16.831 INFO:teuthology.orchestra.run.smithi187.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-30T13:18:16.832 INFO:teuthology.orchestra.run.smithi187.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-30T13:18:16.832 INFO:teuthology.orchestra.run.smithi187.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-03-30T13:18:16.832 INFO:teuthology.orchestra.run.smithi187.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-30T13:18:16.832 INFO:teuthology.orchestra.run.smithi187.stdout:Context: system_u:object_r:device_t:s0 2024-03-30T13:18:16.832 INFO:teuthology.orchestra.run.smithi187.stdout:Access: 2024-03-30 13:18:11.443327898 +0000 2024-03-30T13:18:16.832 INFO:teuthology.orchestra.run.smithi187.stdout:Modify: 2024-03-30 13:13:18.676248832 +0000 2024-03-30T13:18:16.832 INFO:teuthology.orchestra.run.smithi187.stdout:Change: 2024-03-30 13:13:18.676248832 +0000 2024-03-30T13:18:16.832 INFO:teuthology.orchestra.run.smithi187.stdout: Birth: 2024-03-30 13:13:18.676248832 +0000 2024-03-30T13:18:16.832 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-30T13:18:16.901 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records in 2024-03-30T13:18:16.901 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records out 2024-03-30T13:18:16.901 INFO:teuthology.orchestra.run.smithi187.stderr:512 bytes copied, 0.000248977 s, 2.1 MB/s 2024-03-30T13:18:16.902 DEBUG:teuthology.orchestra.run.smithi187:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-30T13:18:16.961 DEBUG:teuthology.orchestra.run.smithi187:> stat /dev/vg_nvme/lv_2 2024-03-30T13:18:17.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:16 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:17.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:16 smithi159 ceph-mon[24787]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:17.017 INFO:teuthology.orchestra.run.smithi187.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-30T13:18:17.018 INFO:teuthology.orchestra.run.smithi187.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-30T13:18:17.018 INFO:teuthology.orchestra.run.smithi187.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-03-30T13:18:17.018 INFO:teuthology.orchestra.run.smithi187.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-30T13:18:17.018 INFO:teuthology.orchestra.run.smithi187.stdout:Context: system_u:object_r:device_t:s0 2024-03-30T13:18:17.018 INFO:teuthology.orchestra.run.smithi187.stdout:Access: 2024-03-30 13:18:11.443327898 +0000 2024-03-30T13:18:17.018 INFO:teuthology.orchestra.run.smithi187.stdout:Modify: 2024-03-30 13:13:19.036242897 +0000 2024-03-30T13:18:17.018 INFO:teuthology.orchestra.run.smithi187.stdout:Change: 2024-03-30 13:13:19.036242897 +0000 2024-03-30T13:18:17.018 INFO:teuthology.orchestra.run.smithi187.stdout: Birth: 2024-03-30 13:13:19.036242897 +0000 2024-03-30T13:18:17.018 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-30T13:18:17.085 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records in 2024-03-30T13:18:17.085 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records out 2024-03-30T13:18:17.085 INFO:teuthology.orchestra.run.smithi187.stderr:512 bytes copied, 0.000224766 s, 2.3 MB/s 2024-03-30T13:18:17.086 DEBUG:teuthology.orchestra.run.smithi187:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-30T13:18:17.144 DEBUG:teuthology.orchestra.run.smithi187:> stat /dev/vg_nvme/lv_3 2024-03-30T13:18:17.199 INFO:teuthology.orchestra.run.smithi187.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-30T13:18:17.199 INFO:teuthology.orchestra.run.smithi187.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-30T13:18:17.199 INFO:teuthology.orchestra.run.smithi187.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-03-30T13:18:17.199 INFO:teuthology.orchestra.run.smithi187.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-30T13:18:17.200 INFO:teuthology.orchestra.run.smithi187.stdout:Context: system_u:object_r:device_t:s0 2024-03-30T13:18:17.200 INFO:teuthology.orchestra.run.smithi187.stdout:Access: 2024-03-30 13:18:11.443327898 +0000 2024-03-30T13:18:17.200 INFO:teuthology.orchestra.run.smithi187.stdout:Modify: 2024-03-30 13:13:19.376237292 +0000 2024-03-30T13:18:17.200 INFO:teuthology.orchestra.run.smithi187.stdout:Change: 2024-03-30 13:13:19.376237292 +0000 2024-03-30T13:18:17.200 INFO:teuthology.orchestra.run.smithi187.stdout: Birth: 2024-03-30 13:13:19.376237292 +0000 2024-03-30T13:18:17.200 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-30T13:18:17.263 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:16 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:17.264 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:16 smithi187 ceph-mon[27499]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:17.266 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records in 2024-03-30T13:18:17.266 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records out 2024-03-30T13:18:17.266 INFO:teuthology.orchestra.run.smithi187.stderr:512 bytes copied, 0.000223179 s, 2.3 MB/s 2024-03-30T13:18:17.267 DEBUG:teuthology.orchestra.run.smithi187:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-30T13:18:17.324 DEBUG:teuthology.orchestra.run.smithi187:> stat /dev/vg_nvme/lv_4 2024-03-30T13:18:17.380 INFO:teuthology.orchestra.run.smithi187.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-30T13:18:17.380 INFO:teuthology.orchestra.run.smithi187.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-30T13:18:17.380 INFO:teuthology.orchestra.run.smithi187.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-03-30T13:18:17.380 INFO:teuthology.orchestra.run.smithi187.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-30T13:18:17.380 INFO:teuthology.orchestra.run.smithi187.stdout:Context: system_u:object_r:device_t:s0 2024-03-30T13:18:17.380 INFO:teuthology.orchestra.run.smithi187.stdout:Access: 2024-03-30 13:18:11.444327881 +0000 2024-03-30T13:18:17.381 INFO:teuthology.orchestra.run.smithi187.stdout:Modify: 2024-03-30 13:13:19.704231884 +0000 2024-03-30T13:18:17.381 INFO:teuthology.orchestra.run.smithi187.stdout:Change: 2024-03-30 13:13:19.704231884 +0000 2024-03-30T13:18:17.381 INFO:teuthology.orchestra.run.smithi187.stdout: Birth: 2024-03-30 13:13:19.704231884 +0000 2024-03-30T13:18:17.381 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-30T13:18:17.446 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records in 2024-03-30T13:18:17.446 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records out 2024-03-30T13:18:17.446 INFO:teuthology.orchestra.run.smithi187.stderr:512 bytes copied, 0.000198972 s, 2.6 MB/s 2024-03-30T13:18:17.447 DEBUG:teuthology.orchestra.run.smithi187:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-30T13:18:17.504 INFO:tasks.cephadm:Deploying osd.0 on smithi159 with /dev/vg_nvme/lv_4... 2024-03-30T13:18:17.504 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-30T13:18:19.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:19 smithi159 ceph-mon[24787]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:19.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:19 smithi187 ceph-mon[27499]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:21.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:21 smithi159 ceph-mon[24787]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:21 smithi187 ceph-mon[27499]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:22.548 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-30T13:18:22.548 INFO:teuthology.orchestra.run.smithi159.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-30T13:18:22.548 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10+0 records in 2024-03-30T13:18:22.548 INFO:teuthology.orchestra.run.smithi159.stderr:10+0 records out 2024-03-30T13:18:22.548 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.01397 s, 751 MB/s 2024-03-30T13:18:22.548 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping successful for: 2024-03-30T13:18:23.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:23 smithi159 ceph-mon[24787]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:23 smithi187 ceph-mon[27499]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:23.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi159:vg_nvme/lv_4 2024-03-30T13:18:24.607 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:24 smithi159 ceph-mon[24787]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:24.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:24 smithi187 ceph-mon[27499]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:27.424 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:27 smithi159 ceph-mon[24787]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:27.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:27 smithi187 ceph-mon[27499]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:28.194 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:18:28.194 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:18:28.194 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:28.194 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:28.194 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:18:28.194 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:28.194 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:18:28.390 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:18:28.390 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:18:28.390 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:28.390 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:28.390 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:18:28.390 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:28.390 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:18:29.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:29 smithi159 ceph-mon[24787]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:18:29.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:29 smithi159 ceph-mon[24787]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:29.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:29 smithi187 ceph-mon[27499]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:18:29.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:29 smithi187 ceph-mon[27499]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:30.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2278953413' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "15f9bf93-f28e-4ef1-a9f2-37a4ed2b0296"}]: dispatch 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2278953413' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "15f9bf93-f28e-4ef1-a9f2-37a4ed2b0296"}]': finished 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: osdmap e6: 1 total, 0 up, 1 in 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:30.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:30 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2683786921' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/2278953413' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "15f9bf93-f28e-4ef1-a9f2-37a4ed2b0296"}]: dispatch 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/2278953413' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "15f9bf93-f28e-4ef1-a9f2-37a4ed2b0296"}]': finished 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: osdmap e6: 1 total, 0 up, 1 in 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:31.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:30 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/2683786921' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:18:32.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:31 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:32.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:31 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:32.909 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:32 smithi159 ceph-mon[24787]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:32.909 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:32 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:33.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:32 smithi187 ceph-mon[27499]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:33.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:32 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:34.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:34 smithi159 ceph-mon[24787]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:18:34.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:34 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:34.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:34 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:34.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:34 smithi159 ceph-mon[24787]: Reconfiguring mon.smithi159 (unknown last config time)... 2024-03-30T13:18:34.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:34 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:18:34.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:34 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:18:34.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:34 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:34.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:34 smithi159 ceph-mon[24787]: Reconfiguring daemon mon.smithi159 on smithi159 2024-03-30T13:18:34.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:34 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:18:34.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:34 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:34.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:34 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:34.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:34 smithi187 ceph-mon[27499]: Reconfiguring mon.smithi159 (unknown last config time)... 2024-03-30T13:18:34.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:34 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:18:34.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:34 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:18:34.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:34 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:34.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:34 smithi187 ceph-mon[27499]: Reconfiguring daemon mon.smithi159 on smithi159 2024-03-30T13:18:35.397 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:35 smithi159 ceph-mon[24787]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:35.397 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:35.397 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi159.fnasly", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:18:35.397 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:18:35.397 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:35.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:35 smithi187 ceph-mon[27499]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:35.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:35.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi159.fnasly", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:18:35.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:18:35.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:36.248 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:36 smithi159 ceph-mon[24787]: Reconfiguring mgr.smithi159.fnasly (unknown last config time)... 2024-03-30T13:18:36.248 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:36 smithi159 ceph-mon[24787]: Reconfiguring daemon mgr.smithi159.fnasly on smithi159 2024-03-30T13:18:36.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:36 smithi187 ceph-mon[27499]: Reconfiguring mgr.smithi159.fnasly (unknown last config time)... 2024-03-30T13:18:36.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:36 smithi187 ceph-mon[27499]: Reconfiguring daemon mgr.smithi159.fnasly on smithi159 2024-03-30T13:18:37.183 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:37 smithi159 ceph-mon[24787]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:37.183 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:37 smithi187 ceph-mon[27499]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:38.338 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:38 smithi159 ceph-mon[24787]: Reconfiguring alertmanager.smithi159 (dependencies changed)... 2024-03-30T13:18:38.338 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:38 smithi159 ceph-mon[24787]: Reconfiguring daemon alertmanager.smithi159 on smithi159 2024-03-30T13:18:38.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:38 smithi187 ceph-mon[27499]: Reconfiguring alertmanager.smithi159 (dependencies changed)... 2024-03-30T13:18:38.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:38 smithi187 ceph-mon[27499]: Reconfiguring daemon alertmanager.smithi159 on smithi159 2024-03-30T13:18:39.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:39 smithi159 ceph-mon[24787]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:39.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:39 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-30T13:18:39.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:39 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:39.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:39 smithi159 ceph-mon[24787]: Deploying daemon osd.0 on smithi159 2024-03-30T13:18:39.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:39 smithi187 ceph-mon[27499]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:39.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:39 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-30T13:18:39.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:39 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:39.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:39 smithi187 ceph-mon[27499]: Deploying daemon osd.0 on smithi159 2024-03-30T13:18:41.259 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:41 smithi159 ceph-mon[24787]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:41.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:41 smithi187 ceph-mon[27499]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:43.395 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:43 smithi159 ceph-mon[24787]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:43.395 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:43 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:43.395 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:43 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi159", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:18:43.395 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:43 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:43.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:43 smithi187 ceph-mon[27499]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:43.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:43 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:43.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:43 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi159", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:18:43.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:43 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:44.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:44 smithi159 ceph-mon[24787]: Reconfiguring crash.smithi159 (monmap changed)... 2024-03-30T13:18:44.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:44 smithi159 ceph-mon[24787]: Reconfiguring daemon crash.smithi159 on smithi159 2024-03-30T13:18:44.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:44 smithi187 ceph-mon[27499]: Reconfiguring crash.smithi159 (monmap changed)... 2024-03-30T13:18:44.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:44 smithi187 ceph-mon[27499]: Reconfiguring daemon crash.smithi159 on smithi159 2024-03-30T13:18:45.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:45 smithi159 ceph-mon[24787]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:45.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:45 smithi187 ceph-mon[27499]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:47.363 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:47 smithi159 ceph-mon[24787]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:47.364 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:18:47.364 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:18:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:47 smithi187 ceph-mon[27499]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:47.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:18:47.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:18:48.730 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:48 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:48.730 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:48 smithi159 ceph-mon[24787]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:49.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:48 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:49.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:48 smithi187 ceph-mon[27499]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:51.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:50 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:51.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:50 smithi159 ceph-mon[24787]: Reconfiguring grafana.smithi159 (dependencies changed)... 2024-03-30T13:18:51.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:50 smithi159 ceph-mon[24787]: Reconfiguring daemon grafana.smithi159 on smithi159 2024-03-30T13:18:51.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:50 smithi159 ceph-mon[24787]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:51.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:50 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:51.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:50 smithi187 ceph-mon[27499]: Reconfiguring grafana.smithi159 (dependencies changed)... 2024-03-30T13:18:51.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:50 smithi187 ceph-mon[27499]: Reconfiguring daemon grafana.smithi159 on smithi159 2024-03-30T13:18:51.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:50 smithi187 ceph-mon[27499]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:51.519 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 0 on host 'smithi159' 2024-03-30T13:18:51.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:51 smithi159 ceph-mon[24787]: from='osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-30T13:18:51.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:52.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:51 smithi187 ceph-mon[27499]: from='osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-30T13:18:52.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:51 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:52.375 DEBUG:teuthology.orchestra.run.smithi159:osd.0> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.0.service 2024-03-30T13:18:52.377 INFO:tasks.cephadm:Deploying osd.1 on smithi159 with /dev/vg_nvme/lv_3... 2024-03-30T13:18:52.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-30T13:18:53.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:52 smithi159 ceph-mon[24787]: from='osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-30T13:18:53.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:52 smithi159 ceph-mon[24787]: osdmap e7: 1 total, 0 up, 1 in 2024-03-30T13:18:53.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:52 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:53.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:52 smithi159 ceph-mon[24787]: from='osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:18:53.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:52 smithi159 ceph-mon[24787]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:53.005 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:18:52 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[35130]: 2024-03-30T13:18:52.742+0000 7f9e0a815700 -1 osd.0 0 waiting for initial osdmap 2024-03-30T13:18:53.005 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:18:52 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[35130]: 2024-03-30T13:18:52.745+0000 7f9e061ae700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:18:53.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:52 smithi187 ceph-mon[27499]: from='osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-30T13:18:53.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:52 smithi187 ceph-mon[27499]: osdmap e7: 1 total, 0 up, 1 in 2024-03-30T13:18:53.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:52 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:53.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:52 smithi187 ceph-mon[27499]: from='osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:18:53.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:52 smithi187 ceph-mon[27499]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:54.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:53 smithi159 ceph-mon[24787]: purged_snaps scrub starts 2024-03-30T13:18:54.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:53 smithi159 ceph-mon[24787]: purged_snaps scrub ok 2024-03-30T13:18:54.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:53 smithi159 ceph-mon[24787]: from='osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-03-30T13:18:54.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:53 smithi159 ceph-mon[24787]: osdmap e8: 1 total, 0 up, 1 in 2024-03-30T13:18:54.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:53 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:54.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:53 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:53 smithi187 ceph-mon[27499]: purged_snaps scrub starts 2024-03-30T13:18:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:53 smithi187 ceph-mon[27499]: purged_snaps scrub ok 2024-03-30T13:18:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:53 smithi187 ceph-mon[27499]: from='osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-03-30T13:18:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:53 smithi187 ceph-mon[27499]: osdmap e8: 1 total, 0 up, 1 in 2024-03-30T13:18:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:53 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:54.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:53 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:55.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:55.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:54 smithi159 ceph-mon[24787]: osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808] boot 2024-03-30T13:18:55.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:54 smithi159 ceph-mon[24787]: osdmap e9: 1 total, 1 up, 1 in 2024-03-30T13:18:55.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:55.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:54 smithi159 ceph-mon[24787]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:55.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:54 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:55.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:54 smithi187 ceph-mon[27499]: osd.0 [v2:172.21.15.159:6802/1641385808,v1:172.21.15.159:6803/1641385808] boot 2024-03-30T13:18:55.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:54 smithi187 ceph-mon[27499]: osdmap e9: 1 total, 1 up, 1 in 2024-03-30T13:18:55.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:54 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:18:55.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:54 smithi187 ceph-mon[27499]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-30T13:18:57.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:56 smithi159 ceph-mon[24787]: osdmap e10: 1 total, 1 up, 1 in 2024-03-30T13:18:57.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:56 smithi159 ceph-mon[24787]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:18:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:56 smithi187 ceph-mon[27499]: osdmap e10: 1 total, 1 up, 1 in 2024-03-30T13:18:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:56 smithi187 ceph-mon[27499]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:18:58.024 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-30T13:18:58.024 INFO:teuthology.orchestra.run.smithi159.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-30T13:18:58.024 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10+0 records in 2024-03-30T13:18:58.024 INFO:teuthology.orchestra.run.smithi159.stderr:10+0 records out 2024-03-30T13:18:58.024 INFO:teuthology.orchestra.run.smithi159.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0134068 s, 782 MB/s 2024-03-30T13:18:58.024 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping successful for: 2024-03-30T13:18:58.926 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi159:vg_nvme/lv_3 2024-03-30T13:18:59.227 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:59 smithi187 ceph-mon[27499]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:18:59.227 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:59 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:59.227 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:59 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:18:59.227 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:18:59 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:18:59.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:59 smithi159 ceph-mon[24787]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:18:59.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:18:59.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:18:59.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:18:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:00.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:00 smithi159 ceph-mon[24787]: Reconfiguring crash.smithi187 (monmap changed)... 2024-03-30T13:19:00.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:00 smithi159 ceph-mon[24787]: Reconfiguring daemon crash.smithi187 on smithi187 2024-03-30T13:19:00.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:00 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:00.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:00 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:19:00.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:00 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:19:00.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:00 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:00.415 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:00 smithi187 ceph-mon[27499]: Reconfiguring crash.smithi187 (monmap changed)... 2024-03-30T13:19:00.415 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:00 smithi187 ceph-mon[27499]: Reconfiguring daemon crash.smithi187 on smithi187 2024-03-30T13:19:00.416 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:00 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:00.416 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:00 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:19:00.416 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:00 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:19:00.416 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:00 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:01.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: Reconfiguring mgr.smithi187.jpkaud (monmap changed)... 2024-03-30T13:19:01.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: Reconfiguring daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:19:01.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:01.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:19:01.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:19:01.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:01.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:01.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:19:01.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:19:01.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:01.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: Reconfiguring mgr.smithi187.jpkaud (monmap changed)... 2024-03-30T13:19:01.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: Reconfiguring daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:19:01.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:01.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:19:01.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:19:01.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:01.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:01.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:19:01.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:19:01.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:02.317 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:02 smithi159 ceph-mon[24787]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:19:02.317 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:02 smithi159 ceph-mon[24787]: Reconfiguring mon.smithi187 (monmap changed)... 2024-03-30T13:19:02.318 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:02 smithi159 ceph-mon[24787]: Reconfiguring daemon mon.smithi187 on smithi187 2024-03-30T13:19:02.341 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:02 smithi187 ceph-mon[27499]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:19:02.341 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:02 smithi187 ceph-mon[27499]: Reconfiguring mon.smithi187 (monmap changed)... 2024-03-30T13:19:02.341 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:02 smithi187 ceph-mon[27499]: Reconfiguring daemon mon.smithi187 on smithi187 2024-03-30T13:19:03.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:03 smithi159 ceph-mon[24787]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:03.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:03 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:03.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:03 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:19:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:03 smithi187 ceph-mon[27499]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:03 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:03 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:19:04.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:04 smithi159 ceph-mon[24787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:19:04.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:04 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1398293194' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5e3306c-4090-412b-ab16-26224afae769"}]: dispatch 2024-03-30T13:19:04.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:04 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1398293194' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5e3306c-4090-412b-ab16-26224afae769"}]': finished 2024-03-30T13:19:04.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:04 smithi159 ceph-mon[24787]: osdmap e11: 2 total, 1 up, 2 in 2024-03-30T13:19:04.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:04.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:04 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/980156811' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:19:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:04 smithi187 ceph-mon[27499]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:19:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:04 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1398293194' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5e3306c-4090-412b-ab16-26224afae769"}]: dispatch 2024-03-30T13:19:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:04 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1398293194' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5e3306c-4090-412b-ab16-26224afae769"}]': finished 2024-03-30T13:19:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:04 smithi187 ceph-mon[27499]: osdmap e11: 2 total, 1 up, 2 in 2024-03-30T13:19:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:04 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/980156811' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:19:05.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:05 smithi159 ceph-mon[24787]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:05 smithi187 ceph-mon[27499]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:07.392 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:07 smithi159 ceph-mon[24787]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:07.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:07 smithi187 ceph-mon[27499]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:09.402 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:09 smithi159 ceph-mon[24787]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:09.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:09 smithi187 ceph-mon[27499]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:10.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-30T13:19:10.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:10 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:10.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-30T13:19:10.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:10 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:11.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:11 smithi159 ceph-mon[24787]: Deploying daemon osd.1 on smithi159 2024-03-30T13:19:11.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:11 smithi159 ceph-mon[24787]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:11.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:11 smithi187 ceph-mon[27499]: Deploying daemon osd.1 on smithi159 2024-03-30T13:19:11.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:11 smithi187 ceph-mon[27499]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:13.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:13 smithi159 ceph-mon[24787]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:13.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:13 smithi187 ceph-mon[27499]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:14.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:14 smithi159 ceph-mon[24787]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:14.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:14 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:14.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:14 smithi187 ceph-mon[27499]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:14.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:14 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:17.249 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:17 smithi159 ceph-mon[24787]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:17.284 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 1 on host 'smithi159' 2024-03-30T13:19:17.378 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:17 smithi187 ceph-mon[27499]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:17.911 DEBUG:teuthology.orchestra.run.smithi159:osd.1> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.1.service 2024-03-30T13:19:17.913 INFO:tasks.cephadm:Deploying osd.2 on smithi159 with /dev/vg_nvme/lv_2... 2024-03-30T13:19:17.913 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-30T13:19:18.523 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:18.523 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.159:9093"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.159:9093"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.159:3000"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.159:3000"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.159:9095"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.159:9095"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:19:18.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:18.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:19:18.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: from='osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-30T13:19:18.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:18 smithi159 ceph-mon[24787]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:18.627 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:18.627 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.159:9093"}]: dispatch 2024-03-30T13:19:18.627 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.159:9093"}]: dispatch 2024-03-30T13:19:18.627 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:18.627 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:19:18.627 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.159:3000"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.159:3000"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.159:9095"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.159:9095"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: from='osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-30T13:19:18.628 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:18 smithi187 ceph-mon[27499]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:19.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:19 smithi159 ceph-mon[24787]: from='osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-30T13:19:19.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:19 smithi159 ceph-mon[24787]: osdmap e12: 2 total, 1 up, 2 in 2024-03-30T13:19:19.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:19 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:19.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:19 smithi159 ceph-mon[24787]: from='osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:19:19.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:19 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:19.505 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:19:19 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[37995]: 2024-03-30T13:19:19.335+0000 7f6d38ca7700 -1 osd.1 0 waiting for initial osdmap 2024-03-30T13:19:19.505 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:19:19 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[37995]: 2024-03-30T13:19:19.338+0000 7f6d32e3d700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:19:19.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:19 smithi187 ceph-mon[27499]: from='osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-30T13:19:19.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:19 smithi187 ceph-mon[27499]: osdmap e12: 2 total, 1 up, 2 in 2024-03-30T13:19:19.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:19 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:19.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:19 smithi187 ceph-mon[27499]: from='osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:19:19.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:19 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:20.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:20 smithi159 ceph-mon[24787]: from='osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-03-30T13:19:20.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:20 smithi159 ceph-mon[24787]: osdmap e13: 2 total, 1 up, 2 in 2024-03-30T13:19:20.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:20 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:20.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:20 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:20.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:20 smithi159 ceph-mon[24787]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:20.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:20 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:20.611 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-30T13:19:20.611 INFO:teuthology.orchestra.run.smithi159.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-30T13:19:20.611 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10+0 records in 2024-03-30T13:19:20.611 INFO:teuthology.orchestra.run.smithi159.stderr:10+0 records out 2024-03-30T13:19:20.611 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.014348 s, 731 MB/s 2024-03-30T13:19:20.612 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping successful for: 2024-03-30T13:19:20.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:20 smithi187 ceph-mon[27499]: from='osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-03-30T13:19:20.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:20 smithi187 ceph-mon[27499]: osdmap e13: 2 total, 1 up, 2 in 2024-03-30T13:19:20.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:20 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:20.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:20 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:20.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:20 smithi187 ceph-mon[27499]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-30T13:19:20.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:20 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:21.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:21 smithi159 ceph-mon[24787]: purged_snaps scrub starts 2024-03-30T13:19:21.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:21 smithi159 ceph-mon[24787]: purged_snaps scrub ok 2024-03-30T13:19:21.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:21 smithi159 ceph-mon[24787]: osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563] boot 2024-03-30T13:19:21.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:21 smithi159 ceph-mon[24787]: osdmap e14: 2 total, 2 up, 2 in 2024-03-30T13:19:21.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:21 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:21.605 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi159:vg_nvme/lv_2 2024-03-30T13:19:21.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:21 smithi187 ceph-mon[27499]: purged_snaps scrub starts 2024-03-30T13:19:21.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:21 smithi187 ceph-mon[27499]: purged_snaps scrub ok 2024-03-30T13:19:21.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:21 smithi187 ceph-mon[27499]: osd.1 [v2:172.21.15.159:6810/2880408563,v1:172.21.15.159:6811/2880408563] boot 2024-03-30T13:19:21.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:21 smithi187 ceph-mon[27499]: osdmap e14: 2 total, 2 up, 2 in 2024-03-30T13:19:21.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:21 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:19:22.663 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:22 smithi159 ceph-mon[24787]: osdmap e15: 2 total, 2 up, 2 in 2024-03-30T13:19:22.663 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:22 smithi159 ceph-mon[24787]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:22.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:22 smithi187 ceph-mon[27499]: osdmap e15: 2 total, 2 up, 2 in 2024-03-30T13:19:22.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:22 smithi187 ceph-mon[27499]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:24.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:24 smithi159 ceph-mon[24787]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:24.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:24 smithi187 ceph-mon[27499]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:25 smithi159 ceph-mon[24787]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:19:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:25 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:19:25.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:25 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:19:25.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:25 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:25.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:25 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:25 smithi187 ceph-mon[27499]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:19:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:25 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:19:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:25 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:19:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:25 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:25 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:26.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:26 smithi159 ceph-mon[24787]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:26.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:26 smithi187 ceph-mon[27499]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:27.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:27 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1052125761' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6ddd12f2-b128-4871-b107-4cc27d6453ba"}]: dispatch 2024-03-30T13:19:27.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:27 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1052125761' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6ddd12f2-b128-4871-b107-4cc27d6453ba"}]': finished 2024-03-30T13:19:27.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:27 smithi159 ceph-mon[24787]: osdmap e16: 3 total, 2 up, 3 in 2024-03-30T13:19:27.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:27.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:27 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1052125761' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6ddd12f2-b128-4871-b107-4cc27d6453ba"}]: dispatch 2024-03-30T13:19:27.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:27 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1052125761' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6ddd12f2-b128-4871-b107-4cc27d6453ba"}]': finished 2024-03-30T13:19:27.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:27 smithi187 ceph-mon[27499]: osdmap e16: 3 total, 2 up, 3 in 2024-03-30T13:19:27.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:28.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:28 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1826054649' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:19:28.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:28 smithi159 ceph-mon[24787]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:28 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1826054649' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:19:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:28 smithi187 ceph-mon[27499]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:31.361 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:31 smithi159 ceph-mon[24787]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:31.476 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:31 smithi187 ceph-mon[27499]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:33.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:19:33.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:33.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:33.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:19:33.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:19:33.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:33.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:33.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:19:33.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:33.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:19:34.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:34 smithi159 ceph-mon[24787]: Detected new or changed devices on smithi159 2024-03-30T13:19:34.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:34 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi159 to 6432M 2024-03-30T13:19:34.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:34 smithi187 ceph-mon[27499]: Detected new or changed devices on smithi159 2024-03-30T13:19:34.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:34 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi159 to 6432M 2024-03-30T13:19:35.402 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:35 smithi159 ceph-mon[24787]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:35.402 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:35.402 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-30T13:19:35.402 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:35.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:35 smithi187 ceph-mon[27499]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:35.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:35.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-30T13:19:35.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:36.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:36 smithi187 ceph-mon[27499]: Deploying daemon osd.2 on smithi159 2024-03-30T13:19:36.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:36 smithi159 ceph-mon[24787]: Deploying daemon osd.2 on smithi159 2024-03-30T13:19:37.493 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:37 smithi159 ceph-mon[24787]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:37 smithi187 ceph-mon[27499]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:38.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:38 smithi159 ceph-mon[24787]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:38.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:38 smithi187 ceph-mon[27499]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:41.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:41 smithi159 ceph-mon[24787]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:41.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:41 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:41.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:41 smithi187 ceph-mon[27499]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:41.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:41 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:43.375 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:43 smithi159 ceph-mon[24787]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:43.375 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:43 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:43.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:43 smithi187 ceph-mon[27499]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:43.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:43 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:45.381 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:45 smithi159 ceph-mon[24787]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:45.381 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:45 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:45.381 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:45 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:45.381 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:45 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:19:45.381 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:45 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:45.382 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:45 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:19:45.420 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:45 smithi187 ceph-mon[27499]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:45.420 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:45 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:45.420 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:45 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:45.420 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:45 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:19:45.420 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:45 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:45.420 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:45 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:19:46.098 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 2 on host 'smithi159' 2024-03-30T13:19:46.402 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:46 smithi159 ceph-mon[24787]: from='osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-30T13:19:46.402 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:46.403 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:46.403 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:19:46.403 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:19:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:46 smithi187 ceph-mon[27499]: from='osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-30T13:19:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:19:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:19:46.684 DEBUG:teuthology.orchestra.run.smithi159:osd.2> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.2.service 2024-03-30T13:19:46.687 INFO:tasks.cephadm:Deploying osd.3 on smithi159 with /dev/vg_nvme/lv_1... 2024-03-30T13:19:46.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-30T13:19:47.441 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:47 smithi159 ceph-mon[24787]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:47.442 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:47 smithi159 ceph-mon[24787]: from='osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-30T13:19:47.442 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:47 smithi159 ceph-mon[24787]: osdmap e17: 3 total, 2 up, 3 in 2024-03-30T13:19:47.442 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:47.442 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:47 smithi159 ceph-mon[24787]: from='osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:19:47.442 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:19:47 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[41867]: 2024-03-30T13:19:47.152+0000 7f0e5b6f7700 -1 osd.2 0 waiting for initial osdmap 2024-03-30T13:19:47.442 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:19:47 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[41867]: 2024-03-30T13:19:47.155+0000 7f0e5688f700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:19:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:47 smithi187 ceph-mon[27499]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:47 smithi187 ceph-mon[27499]: from='osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-30T13:19:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:47 smithi187 ceph-mon[27499]: osdmap e17: 3 total, 2 up, 3 in 2024-03-30T13:19:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:47 smithi187 ceph-mon[27499]: from='osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:19:48.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:48 smithi159 ceph-mon[24787]: from='osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-03-30T13:19:48.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:48 smithi159 ceph-mon[24787]: osdmap e18: 3 total, 2 up, 3 in 2024-03-30T13:19:48.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:48 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:48.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:48 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:48.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:48 smithi187 ceph-mon[27499]: from='osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-03-30T13:19:48.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:48 smithi187 ceph-mon[27499]: osdmap e18: 3 total, 2 up, 3 in 2024-03-30T13:19:48.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:48 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:48.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:48 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:49.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:49 smithi159 ceph-mon[24787]: purged_snaps scrub starts 2024-03-30T13:19:49.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:49 smithi159 ceph-mon[24787]: purged_snaps scrub ok 2024-03-30T13:19:49.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:49 smithi159 ceph-mon[24787]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:49.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:49.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:49 smithi159 ceph-mon[24787]: osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591] boot 2024-03-30T13:19:49.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:49 smithi159 ceph-mon[24787]: osdmap e19: 3 total, 3 up, 3 in 2024-03-30T13:19:49.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:49.519 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-30T13:19:49.520 INFO:teuthology.orchestra.run.smithi159.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-30T13:19:49.520 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10+0 records in 2024-03-30T13:19:49.520 INFO:teuthology.orchestra.run.smithi159.stderr:10+0 records out 2024-03-30T13:19:49.520 INFO:teuthology.orchestra.run.smithi159.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0150348 s, 697 MB/s 2024-03-30T13:19:49.520 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping successful for: 2024-03-30T13:19:49.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:49 smithi187 ceph-mon[27499]: purged_snaps scrub starts 2024-03-30T13:19:49.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:49 smithi187 ceph-mon[27499]: purged_snaps scrub ok 2024-03-30T13:19:49.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:49 smithi187 ceph-mon[27499]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-30T13:19:49.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:49.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:49 smithi187 ceph-mon[27499]: osd.2 [v2:172.21.15.159:6818/1039946591,v1:172.21.15.159:6819/1039946591] boot 2024-03-30T13:19:49.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:49 smithi187 ceph-mon[27499]: osdmap e19: 3 total, 3 up, 3 in 2024-03-30T13:19:49.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:19:50.273 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi159:vg_nvme/lv_1 2024-03-30T13:19:50.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:50 smithi159 ceph-mon[24787]: osdmap e20: 3 total, 3 up, 3 in 2024-03-30T13:19:50.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:50 smithi187 ceph-mon[27499]: osdmap e20: 3 total, 3 up, 3 in 2024-03-30T13:19:51.726 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:51 smithi159 ceph-mon[24787]: pgmap v70: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:51.726 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-30T13:19:51.726 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-30T13:19:51.726 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:51 smithi159 ceph-mon[24787]: osdmap e21: 3 total, 3 up, 3 in 2024-03-30T13:19:51.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-30T13:19:51.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:51 smithi187 ceph-mon[27499]: pgmap v70: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:51.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:51 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-30T13:19:51.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:51 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-30T13:19:51.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:51 smithi187 ceph-mon[27499]: osdmap e21: 3 total, 3 up, 3 in 2024-03-30T13:19:51.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:51 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-30T13:19:52.683 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:52 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-30T13:19:52.683 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:52 smithi159 ceph-mon[24787]: osdmap e22: 3 total, 3 up, 3 in 2024-03-30T13:19:52.683 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:52 smithi159 ceph-mon[24787]: pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:52.683 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:19:52 smithi159 sudo[43299]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-30T13:19:52.683 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:19:52 smithi159 sudo[43299]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:52.683 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:19:52 smithi159 sudo[43299]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:52.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:52 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-30T13:19:52.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:52 smithi187 ceph-mon[27499]: osdmap e22: 3 total, 3 up, 3 in 2024-03-30T13:19:52.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:52 smithi187 ceph-mon[27499]: pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:52.996 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:19:52 smithi159 sudo[43325]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-30T13:19:52.996 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:19:52 smithi159 sudo[43325]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:52.996 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:19:52 smithi159 sudo[43325]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:53.254 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43392]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-30T13:19:53.254 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43392]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:53.254 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43392]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:53.697 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43408]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-30T13:19:53.697 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43408]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:53.697 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43408]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:53.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:53 smithi159 ceph-mon[24787]: osdmap e23: 3 total, 3 up, 3 in 2024-03-30T13:19:53.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:53 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:53.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:53 smithi159 ceph-mon[24787]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:19:53.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:53 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:19:53.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:53 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:19:53.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:53 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:53.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:53 smithi187 ceph-mon[27499]: osdmap e23: 3 total, 3 up, 3 in 2024-03-30T13:19:53.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:53 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:53.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:53 smithi187 ceph-mon[27499]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:19:53.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:53 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:19:53.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:53 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:19:53.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:53 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:19:53.958 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43474]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-30T13:19:53.958 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43474]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:53.958 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43474]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:54.217 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 sudo[43514]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-30T13:19:54.217 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 sudo[43514]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:54.217 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 sudo[43514]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:54.217 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43511]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-30T13:19:54.218 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43511]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:54.218 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:19:53 smithi159 sudo[43511]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:54.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 sudo[43519]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-30T13:19:54.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 sudo[43519]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:54.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 sudo[43519]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:54.957 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 sudo[29822]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-30T13:19:54.957 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 sudo[29822]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:54.957 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 sudo[29822]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:55.055 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 ceph-mon[24787]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-30T13:19:55.056 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 ceph-mon[24787]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:55.056 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 ceph-mon[24787]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-30T13:19:55.056 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:19:55.056 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:19:55.056 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:19:55.056 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:54 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:19:55.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 sudo[29825]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-30T13:19:55.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 sudo[29825]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-30T13:19:55.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 sudo[29825]: pam_unix(sudo:session): session closed for user root 2024-03-30T13:19:55.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 ceph-mon[27499]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-30T13:19:55.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 ceph-mon[27499]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:55.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 ceph-mon[27499]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-30T13:19:55.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:19:55.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:19:55.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:19:55.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:54 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:19:56.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-30T13:19:56.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-30T13:19:56.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: mgrmap e19: smithi159.fnasly(active, since 2m), standbys: smithi187.jpkaud 2024-03-30T13:19:56.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:56.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:56.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3690620741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1e990270-44f5-4f81-b3fc-5ffce2cfd21f"}]: dispatch 2024-03-30T13:19:56.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3690620741' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1e990270-44f5-4f81-b3fc-5ffce2cfd21f"}]': finished 2024-03-30T13:19:56.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: osdmap e24: 4 total, 3 up, 4 in 2024-03-30T13:19:56.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:55 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:19:56.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-30T13:19:56.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-30T13:19:56.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: mgrmap e19: smithi159.fnasly(active, since 2m), standbys: smithi187.jpkaud 2024-03-30T13:19:56.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:56.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:19:56.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3690620741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1e990270-44f5-4f81-b3fc-5ffce2cfd21f"}]: dispatch 2024-03-30T13:19:56.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3690620741' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1e990270-44f5-4f81-b3fc-5ffce2cfd21f"}]': finished 2024-03-30T13:19:56.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: osdmap e24: 4 total, 3 up, 4 in 2024-03-30T13:19:56.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:55 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:19:57.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:56 smithi159 ceph-mon[24787]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:57.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:56 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1950788789' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:19:57.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:56 smithi187 ceph-mon[27499]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:57.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:56 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1950788789' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:19:59.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:19:59 smithi159 ceph-mon[24787]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:19:59.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:19:59 smithi187 ceph-mon[27499]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:00.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:00 smithi159 ceph-mon[24787]: overall HEALTH_OK 2024-03-30T13:20:00.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:00 smithi187 ceph-mon[27499]: overall HEALTH_OK 2024-03-30T13:20:01.424 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:01.424 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:01.424 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:01.424 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:01.424 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:01.424 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:01.424 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:01.424 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:01.425 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:01.425 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:01.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:01.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:01.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:01.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:01.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:01.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:01.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:01.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:01.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:01.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:02.459 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:02 smithi159 ceph-mon[24787]: Detected new or changed devices on smithi159 2024-03-30T13:20:02.459 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:02 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi159 to 4288M 2024-03-30T13:20:02.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:02 smithi187 ceph-mon[27499]: Detected new or changed devices on smithi159 2024-03-30T13:20:02.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:02 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi159 to 4288M 2024-03-30T13:20:03.243 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:03 smithi159 ceph-mon[24787]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:03 smithi187 ceph-mon[27499]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:04.718 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-30T13:20:04.719 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:04.719 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:04 smithi159 ceph-mon[24787]: Deploying daemon osd.3 on smithi159 2024-03-30T13:20:04.719 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:04 smithi159 ceph-mon[24787]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:04.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-30T13:20:04.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:04.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:04 smithi187 ceph-mon[27499]: Deploying daemon osd.3 on smithi159 2024-03-30T13:20:04.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:04 smithi187 ceph-mon[27499]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:07.400 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:07 smithi159 ceph-mon[24787]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:07.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:07 smithi187 ceph-mon[27499]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:08 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:08 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:08 smithi159 ceph-mon[24787]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:09.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:08 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:09.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:08 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:09.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:08 smithi187 ceph-mon[27499]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:11.266 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:11 smithi159 ceph-mon[24787]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:11.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:11 smithi187 ceph-mon[27499]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:12.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:12 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:12.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:12 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:12.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:12 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:12.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:12 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:12.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:12 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:12.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:12 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:12.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:12 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:12.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:12 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:13.444 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:13 smithi159 ceph-mon[24787]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:13.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:13 smithi187 ceph-mon[27499]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:14.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:14 smithi159 ceph-mon[24787]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:14.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:14 smithi159 ceph-mon[24787]: from='osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-30T13:20:14.734 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 3 on host 'smithi159' 2024-03-30T13:20:14.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:14 smithi187 ceph-mon[27499]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:14.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:14 smithi187 ceph-mon[27499]: from='osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-30T13:20:15.364 DEBUG:teuthology.orchestra.run.smithi159:osd.3> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.3.service 2024-03-30T13:20:15.367 INFO:tasks.cephadm:Deploying osd.4 on smithi187 with /dev/vg_nvme/lv_4... 2024-03-30T13:20:15.367 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-30T13:20:15.678 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:15 smithi187 ceph-mon[27499]: from='osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-30T13:20:15.678 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:15 smithi187 ceph-mon[27499]: osdmap e25: 4 total, 3 up, 4 in 2024-03-30T13:20:15.678 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:15 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:20:15.678 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:15 smithi187 ceph-mon[27499]: from='osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:20:15.678 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:15 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:15.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:15 smithi159 ceph-mon[24787]: from='osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-30T13:20:15.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:15 smithi159 ceph-mon[24787]: osdmap e25: 4 total, 3 up, 4 in 2024-03-30T13:20:15.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:15 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:20:15.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:15 smithi159 ceph-mon[24787]: from='osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:20:15.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:15 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:15.755 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:20:15 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[45730]: 2024-03-30T13:20:15.465+0000 7f0fdd844700 -1 osd.3 0 waiting for initial osdmap 2024-03-30T13:20:15.755 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:20:15 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[45730]: 2024-03-30T13:20:15.469+0000 7f0fda1df700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:20:16.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:16 smithi159 ceph-mon[24787]: from='osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-03-30T13:20:16.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:16 smithi159 ceph-mon[24787]: osdmap e26: 4 total, 3 up, 4 in 2024-03-30T13:20:16.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:16 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:20:16.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:16 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:20:16.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:16 smithi159 ceph-mon[24787]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:16.694 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:16 smithi187 ceph-mon[27499]: from='osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-03-30T13:20:16.694 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:16 smithi187 ceph-mon[27499]: osdmap e26: 4 total, 3 up, 4 in 2024-03-30T13:20:16.694 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:16 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:20:16.694 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:16 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:20:16.695 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:16 smithi187 ceph-mon[27499]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-30T13:20:17.249 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-30T13:20:17.249 INFO:teuthology.orchestra.run.smithi187.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-30T13:20:17.249 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10+0 records in 2024-03-30T13:20:17.249 INFO:teuthology.orchestra.run.smithi187.stderr:10+0 records out 2024-03-30T13:20:17.250 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112959 s, 928 MB/s 2024-03-30T13:20:17.250 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping successful for: 2024-03-30T13:20:17.635 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:17 smithi159 ceph-mon[24787]: purged_snaps scrub starts 2024-03-30T13:20:17.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:17 smithi159 ceph-mon[24787]: purged_snaps scrub ok 2024-03-30T13:20:17.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:17 smithi159 ceph-mon[24787]: osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583] boot 2024-03-30T13:20:17.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:17 smithi159 ceph-mon[24787]: osdmap e27: 4 total, 4 up, 4 in 2024-03-30T13:20:17.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:17 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:20:17.703 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:17 smithi187 ceph-mon[27499]: purged_snaps scrub starts 2024-03-30T13:20:17.703 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:17 smithi187 ceph-mon[27499]: purged_snaps scrub ok 2024-03-30T13:20:17.703 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:17 smithi187 ceph-mon[27499]: osd.3 [v2:172.21.15.159:6826/1390459583,v1:172.21.15.159:6827/1390459583] boot 2024-03-30T13:20:17.703 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:17 smithi187 ceph-mon[27499]: osdmap e27: 4 total, 4 up, 4 in 2024-03-30T13:20:17.703 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:17 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:20:17.821 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi187:vg_nvme/lv_4 2024-03-30T13:20:18.732 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:18 smithi159 ceph-mon[24787]: osdmap e28: 4 total, 4 up, 4 in 2024-03-30T13:20:18.733 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:18.733 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:18 smithi159 ceph-mon[24787]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:18.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:18 smithi187 ceph-mon[27499]: osdmap e28: 4 total, 4 up, 4 in 2024-03-30T13:20:18.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:18.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:18 smithi187 ceph-mon[27499]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:20.217 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:20 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:20:20.217 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:20 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:20:20.217 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:20 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:20.417 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:20 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:20:20.418 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:20 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:20:20.418 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:20 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:21.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:21 smithi187 ceph-mon[27499]: from='client.24153 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:20:21.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:21 smithi187 ceph-mon[27499]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:21.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:21 smithi159 ceph-mon[24787]: from='client.24153 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:20:21.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:21 smithi159 ceph-mon[24787]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:22.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:22.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi159 to 3216M 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/2924394850' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff5b3592-6def-4081-8e9c-4d89e0607324"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff5b3592-6def-4081-8e9c-4d89e0607324"}]: dispatch 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff5b3592-6def-4081-8e9c-4d89e0607324"}]': finished 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: osdmap e29: 5 total, 4 up, 5 in 2024-03-30T13:20:22.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:22.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:22.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:22.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:22.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:22.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:22.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi159 to 3216M 2024-03-30T13:20:22.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/2924394850' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff5b3592-6def-4081-8e9c-4d89e0607324"}]: dispatch 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff5b3592-6def-4081-8e9c-4d89e0607324"}]: dispatch 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff5b3592-6def-4081-8e9c-4d89e0607324"}]': finished 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: osdmap e29: 5 total, 4 up, 5 in 2024-03-30T13:20:22.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:23.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:23 smithi159 ceph-mon[24787]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:23.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:23 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/1015504355' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:20:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:23 smithi187 ceph-mon[27499]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:23 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/1015504355' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:20:24.753 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:24 smithi187 ceph-mon[27499]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:24.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:24 smithi159 ceph-mon[24787]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:26.443 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:26 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:26.444 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:26 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:26.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:26 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:26.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:26 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:27.457 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:27 smithi187 ceph-mon[27499]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:27 smithi159 ceph-mon[24787]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-30T13:20:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:28.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-30T13:20:28.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:29.447 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:29 smithi187 ceph-mon[27499]: Deploying daemon osd.4 on smithi187 2024-03-30T13:20:29.447 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:29 smithi187 ceph-mon[27499]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:29 smithi159 ceph-mon[24787]: Deploying daemon osd.4 on smithi187 2024-03-30T13:20:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:29 smithi159 ceph-mon[24787]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:31.485 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:31 smithi187 ceph-mon[27499]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:31.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:31 smithi159 ceph-mon[24787]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:33.395 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:33 smithi187 ceph-mon[27499]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:33.395 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:33.396 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:33.396 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:33.396 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:33 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:33.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:33 smithi159 ceph-mon[24787]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:33.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:33.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:33.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:33.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:33 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:34.753 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:34 smithi187 ceph-mon[27499]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:34.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:34 smithi159 ceph-mon[24787]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:35.442 INFO:teuthology.orchestra.run.smithi187.stdout:Created osd(s) 4 on host 'smithi187' 2024-03-30T13:20:35.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:35 smithi159 ceph-mon[24787]: from='osd.4 [v2:172.21.15.187:6800/2785191302,v1:172.21.15.187:6801/2785191302]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-30T13:20:35.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:35 smithi159 ceph-mon[24787]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-30T13:20:35.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:35.767 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:35 smithi187 ceph-mon[27499]: from='osd.4 [v2:172.21.15.187:6800/2785191302,v1:172.21.15.187:6801/2785191302]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-30T13:20:35.767 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:35 smithi187 ceph-mon[27499]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-30T13:20:35.767 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:36.094 DEBUG:teuthology.orchestra.run.smithi187:osd.4> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.4.service 2024-03-30T13:20:36.096 INFO:tasks.cephadm:Deploying osd.5 on smithi187 with /dev/vg_nvme/lv_3... 2024-03-30T13:20:36.096 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-30T13:20:36.663 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:36 smithi187 ceph-mon[27499]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-30T13:20:36.663 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:36 smithi187 ceph-mon[27499]: osdmap e30: 5 total, 4 up, 5 in 2024-03-30T13:20:36.663 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:36 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:36.663 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:36 smithi187 ceph-mon[27499]: from='osd.4 [v2:172.21.15.187:6800/2785191302,v1:172.21.15.187:6801/2785191302]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:20:36.664 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:36 smithi187 ceph-mon[27499]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:20:36.664 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:36 smithi187 ceph-mon[27499]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:36.664 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:20:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[31753]: 2024-03-30T13:20:36.474+0000 7f84f45dd700 -1 osd.4 0 waiting for initial osdmap 2024-03-30T13:20:36.664 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:20:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[31753]: 2024-03-30T13:20:36.478+0000 7f84ee773700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:20:36.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:36 smithi159 ceph-mon[24787]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-30T13:20:36.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:36 smithi159 ceph-mon[24787]: osdmap e30: 5 total, 4 up, 5 in 2024-03-30T13:20:36.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:36 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:36.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:36 smithi159 ceph-mon[24787]: from='osd.4 [v2:172.21.15.187:6800/2785191302,v1:172.21.15.187:6801/2785191302]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:20:36.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:36 smithi159 ceph-mon[24787]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:20:36.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:36 smithi159 ceph-mon[24787]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:37.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:37 smithi159 ceph-mon[24787]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-30T13:20:37.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:37 smithi159 ceph-mon[24787]: osdmap e31: 5 total, 4 up, 5 in 2024-03-30T13:20:37.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:37.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:37.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:37.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:37 smithi187 ceph-mon[27499]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-30T13:20:37.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:37 smithi187 ceph-mon[27499]: osdmap e31: 5 total, 4 up, 5 in 2024-03-30T13:20:37.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:37.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:37.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:38.553 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-30T13:20:38.553 INFO:teuthology.orchestra.run.smithi187.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-30T13:20:38.553 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10+0 records in 2024-03-30T13:20:38.553 INFO:teuthology.orchestra.run.smithi187.stderr:10+0 records out 2024-03-30T13:20:38.553 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123752 s, 847 MB/s 2024-03-30T13:20:38.553 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping successful for: 2024-03-30T13:20:38.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:38 smithi159 ceph-mon[24787]: purged_snaps scrub starts 2024-03-30T13:20:38.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:38 smithi159 ceph-mon[24787]: purged_snaps scrub ok 2024-03-30T13:20:38.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:38 smithi159 ceph-mon[24787]: osd.4 [v2:172.21.15.187:6800/2785191302,v1:172.21.15.187:6801/2785191302] boot 2024-03-30T13:20:38.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:38 smithi159 ceph-mon[24787]: osdmap e32: 5 total, 5 up, 5 in 2024-03-30T13:20:38.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:38 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:38.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:38 smithi159 ceph-mon[24787]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:38.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:38 smithi159 ceph-mon[24787]: osdmap e33: 5 total, 5 up, 5 in 2024-03-30T13:20:38.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:38 smithi187 ceph-mon[27499]: purged_snaps scrub starts 2024-03-30T13:20:38.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:38 smithi187 ceph-mon[27499]: purged_snaps scrub ok 2024-03-30T13:20:38.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:38 smithi187 ceph-mon[27499]: osd.4 [v2:172.21.15.187:6800/2785191302,v1:172.21.15.187:6801/2785191302] boot 2024-03-30T13:20:38.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:38 smithi187 ceph-mon[27499]: osdmap e32: 5 total, 5 up, 5 in 2024-03-30T13:20:38.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:38 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:20:38.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:38 smithi187 ceph-mon[27499]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-30T13:20:38.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:38 smithi187 ceph-mon[27499]: osdmap e33: 5 total, 5 up, 5 in 2024-03-30T13:20:39.177 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi187:vg_nvme/lv_3 2024-03-30T13:20:40.515 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:40 smithi187 ceph-mon[27499]: osdmap e34: 5 total, 5 up, 5 in 2024-03-30T13:20:40.516 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:40 smithi187 ceph-mon[27499]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:40.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:40 smithi159 ceph-mon[24787]: osdmap e34: 5 total, 5 up, 5 in 2024-03-30T13:20:40.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:40 smithi159 ceph-mon[24787]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:41.612 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:41 smithi187 ceph-mon[27499]: osdmap e35: 5 total, 5 up, 5 in 2024-03-30T13:20:41.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:41 smithi159 ceph-mon[24787]: osdmap e35: 5 total, 5 up, 5 in 2024-03-30T13:20:42.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: osdmap e36: 5 total, 5 up, 5 in 2024-03-30T13:20:42.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: Detected new or changed devices on smithi187 2024-03-30T13:20:42.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:42.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:42.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi187 to 15936M 2024-03-30T13:20:42.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:42.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:42.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:42.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:42.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:42.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:42.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:42 smithi187 ceph-mon[27499]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-30T13:20:42.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: osdmap e36: 5 total, 5 up, 5 in 2024-03-30T13:20:42.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: Detected new or changed devices on smithi187 2024-03-30T13:20:42.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:42.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:20:42.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi187 to 15936M 2024-03-30T13:20:42.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:42.758 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:42.758 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:42.758 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:42.758 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:42.758 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:42.758 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:42 smithi159 ceph-mon[24787]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-30T13:20:43.713 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:43 smithi187 ceph-mon[27499]: from='client.24171 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:20:43.713 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:43 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:20:43.713 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:43 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:20:43.714 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:43 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:44.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:43 smithi159 ceph-mon[24787]: from='client.24171 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:20:44.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:43 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:20:44.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:43 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:20:44.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:43 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:45.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:44 smithi159 ceph-mon[24787]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-03-30T13:20:45.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:44 smithi187 ceph-mon[27499]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-03-30T13:20:46.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:45 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/868615578' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72f07bc-6453-4ff6-b0bf-a6f9062d5696"}]: dispatch 2024-03-30T13:20:46.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:45 smithi159 ceph-mon[24787]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72f07bc-6453-4ff6-b0bf-a6f9062d5696"}]: dispatch 2024-03-30T13:20:46.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:45 smithi159 ceph-mon[24787]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f72f07bc-6453-4ff6-b0bf-a6f9062d5696"}]': finished 2024-03-30T13:20:46.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:45 smithi159 ceph-mon[24787]: osdmap e37: 6 total, 5 up, 6 in 2024-03-30T13:20:46.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:45 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:20:46.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:45 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/2517665908' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:20:46.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:45 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/868615578' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72f07bc-6453-4ff6-b0bf-a6f9062d5696"}]: dispatch 2024-03-30T13:20:46.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:45 smithi187 ceph-mon[27499]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72f07bc-6453-4ff6-b0bf-a6f9062d5696"}]: dispatch 2024-03-30T13:20:46.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:45 smithi187 ceph-mon[27499]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f72f07bc-6453-4ff6-b0bf-a6f9062d5696"}]': finished 2024-03-30T13:20:46.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:45 smithi187 ceph-mon[27499]: osdmap e37: 6 total, 5 up, 6 in 2024-03-30T13:20:46.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:45 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:20:46.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:45 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/2517665908' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:20:47.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:46 smithi159 ceph-mon[24787]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-30T13:20:47.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:20:47.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:20:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:46 smithi187 ceph-mon[27499]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-30T13:20:47.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:20:47.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:20:49.347 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:49 smithi187 ceph-mon[27499]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-03-30T13:20:49.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:49 smithi159 ceph-mon[24787]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-03-30T13:20:51.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:51 smithi187 ceph-mon[27499]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:51.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:51 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-30T13:20:51.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:51 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:51.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:51 smithi159 ceph-mon[24787]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:51.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-30T13:20:51.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:51 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:52.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:52 smithi159 ceph-mon[24787]: Deploying daemon osd.5 on smithi187 2024-03-30T13:20:52.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:52 smithi187 ceph-mon[27499]: Deploying daemon osd.5 on smithi187 2024-03-30T13:20:53.458 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:53 smithi187 ceph-mon[27499]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:53.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:53 smithi159 ceph-mon[24787]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:54.629 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:54 smithi187 ceph-mon[27499]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:54.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:54 smithi159 ceph-mon[24787]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:56.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:56 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:56.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:56 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:56.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:56 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:56.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:56 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:56.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:56 smithi159 ceph-mon[24787]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:56.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:56 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:56.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:56 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:20:56.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:56 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:20:56.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:56 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:20:56.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:56 smithi187 ceph-mon[27499]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:58.307 INFO:teuthology.orchestra.run.smithi187.stdout:Created osd(s) 5 on host 'smithi187' 2024-03-30T13:20:58.933 DEBUG:teuthology.orchestra.run.smithi187:osd.5> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.5.service 2024-03-30T13:20:58.936 INFO:tasks.cephadm:Deploying osd.6 on smithi187 with /dev/vg_nvme/lv_2... 2024-03-30T13:20:58.936 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-30T13:20:59.229 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:59 smithi187 ceph-mon[27499]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:59.229 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:59 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:59.229 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:59 smithi187 ceph-mon[27499]: from='osd.5 [v2:172.21.15.187:6808/1659074308,v1:172.21.15.187:6809/1659074308]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-30T13:20:59.229 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:20:59 smithi187 ceph-mon[27499]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-30T13:20:59.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:59 smithi159 ceph-mon[24787]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:20:59.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:20:59.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:59 smithi159 ceph-mon[24787]: from='osd.5 [v2:172.21.15.187:6808/1659074308,v1:172.21.15.187:6809/1659074308]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-30T13:20:59.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:20:59 smithi159 ceph-mon[24787]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-30T13:21:00.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:00 smithi159 ceph-mon[24787]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-30T13:21:00.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:00 smithi159 ceph-mon[24787]: osdmap e38: 6 total, 5 up, 6 in 2024-03-30T13:21:00.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:00 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:21:00.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:00 smithi159 ceph-mon[24787]: from='osd.5 [v2:172.21.15.187:6808/1659074308,v1:172.21.15.187:6809/1659074308]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:00.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:00 smithi159 ceph-mon[24787]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:00.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:00 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:00.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:00 smithi159 ceph-mon[24787]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:21:00.760 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-mon[27499]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-30T13:21:00.760 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-mon[27499]: osdmap e38: 6 total, 5 up, 6 in 2024-03-30T13:21:00.760 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:21:00.760 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-mon[27499]: from='osd.5 [v2:172.21.15.187:6808/1659074308,v1:172.21.15.187:6809/1659074308]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:00.761 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-mon[27499]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:00.761 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:00.761 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-mon[27499]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:21:00.761 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[34744]: 2024-03-30T13:21:00.471+0000 7f002b41f700 -1 osd.5 0 waiting for initial osdmap 2024-03-30T13:21:00.761 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:21:00 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[34744]: 2024-03-30T13:21:00.476+0000 7f00245b3700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:21:01.342 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-30T13:21:01.342 INFO:teuthology.orchestra.run.smithi187.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-30T13:21:01.342 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10+0 records in 2024-03-30T13:21:01.342 INFO:teuthology.orchestra.run.smithi187.stderr:10+0 records out 2024-03-30T13:21:01.342 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129172 s, 812 MB/s 2024-03-30T13:21:01.343 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping successful for: 2024-03-30T13:21:01.614 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:01 smithi187 ceph-mon[27499]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-30T13:21:01.614 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:01 smithi187 ceph-mon[27499]: osdmap e39: 6 total, 5 up, 6 in 2024-03-30T13:21:01.614 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:21:01.614 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:01 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:21:01.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:01 smithi159 ceph-mon[24787]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-30T13:21:01.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:01 smithi159 ceph-mon[24787]: osdmap e39: 6 total, 5 up, 6 in 2024-03-30T13:21:01.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:21:01.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:01 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:21:02.365 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi187:vg_nvme/lv_2 2024-03-30T13:21:02.674 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:02 smithi187 ceph-mon[27499]: purged_snaps scrub starts 2024-03-30T13:21:02.674 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:02 smithi187 ceph-mon[27499]: purged_snaps scrub ok 2024-03-30T13:21:02.674 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:02 smithi187 ceph-mon[27499]: osd.5 [v2:172.21.15.187:6808/1659074308,v1:172.21.15.187:6809/1659074308] boot 2024-03-30T13:21:02.675 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:02 smithi187 ceph-mon[27499]: osdmap e40: 6 total, 6 up, 6 in 2024-03-30T13:21:02.675 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:02 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:21:02.675 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:02 smithi187 ceph-mon[27499]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:21:02.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:02 smithi159 ceph-mon[24787]: purged_snaps scrub starts 2024-03-30T13:21:02.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:02 smithi159 ceph-mon[24787]: purged_snaps scrub ok 2024-03-30T13:21:02.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:02 smithi159 ceph-mon[24787]: osd.5 [v2:172.21.15.187:6808/1659074308,v1:172.21.15.187:6809/1659074308] boot 2024-03-30T13:21:02.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:02 smithi159 ceph-mon[24787]: osdmap e40: 6 total, 6 up, 6 in 2024-03-30T13:21:02.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:21:02.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:02 smithi159 ceph-mon[24787]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-30T13:21:03.702 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:03 smithi187 ceph-mon[27499]: osdmap e41: 6 total, 6 up, 6 in 2024-03-30T13:21:03.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:03 smithi159 ceph-mon[24787]: osdmap e41: 6 total, 6 up, 6 in 2024-03-30T13:21:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: osdmap e42: 6 total, 6 up, 6 in 2024-03-30T13:21:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: Detected new or changed devices on smithi187 2024-03-30T13:21:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi187 to 7968M 2024-03-30T13:21:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:04.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:04.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:04.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:04.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:04.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:04 smithi187 ceph-mon[27499]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:04.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: osdmap e42: 6 total, 6 up, 6 in 2024-03-30T13:21:04.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: Detected new or changed devices on smithi187 2024-03-30T13:21:04.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:04.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:04.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:04.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi187 to 7968M 2024-03-30T13:21:04.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:04.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:04.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:04.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:04.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:04.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:04.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:04 smithi159 ceph-mon[24787]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:05.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:05 smithi159 ceph-mon[24787]: osdmap e43: 6 total, 6 up, 6 in 2024-03-30T13:21:05.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:05 smithi159 ceph-mon[24787]: from='client.24189 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:21:05.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:21:05.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:21:05.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:05.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:05 smithi187 ceph-mon[27499]: osdmap e43: 6 total, 6 up, 6 in 2024-03-30T13:21:05.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:05 smithi187 ceph-mon[27499]: from='client.24189 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:21:05.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:21:05.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:21:05.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:06 smithi187 ceph-mon[27499]: osdmap e44: 6 total, 6 up, 6 in 2024-03-30T13:21:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:06 smithi187 ceph-mon[27499]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:06.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:06 smithi159 ceph-mon[24787]: osdmap e44: 6 total, 6 up, 6 in 2024-03-30T13:21:06.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:06 smithi159 ceph-mon[24787]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:07.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:07 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/3155518457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17b28af4-348e-4983-9c5e-d19fb58e0624"}]: dispatch 2024-03-30T13:21:07.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:07 smithi159 ceph-mon[24787]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17b28af4-348e-4983-9c5e-d19fb58e0624"}]: dispatch 2024-03-30T13:21:07.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:07 smithi159 ceph-mon[24787]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17b28af4-348e-4983-9c5e-d19fb58e0624"}]': finished 2024-03-30T13:21:07.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:07 smithi159 ceph-mon[24787]: osdmap e45: 7 total, 6 up, 7 in 2024-03-30T13:21:07.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:07 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:07.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:07 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/3155518457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17b28af4-348e-4983-9c5e-d19fb58e0624"}]: dispatch 2024-03-30T13:21:07.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:07 smithi187 ceph-mon[27499]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17b28af4-348e-4983-9c5e-d19fb58e0624"}]: dispatch 2024-03-30T13:21:07.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:07 smithi187 ceph-mon[27499]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17b28af4-348e-4983-9c5e-d19fb58e0624"}]': finished 2024-03-30T13:21:07.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:07 smithi187 ceph-mon[27499]: osdmap e45: 7 total, 6 up, 7 in 2024-03-30T13:21:07.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:07 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:08.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:08 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/3958755296' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:21:08.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:08 smithi187 ceph-mon[27499]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:09.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:08 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/3958755296' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:21:09.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:08 smithi159 ceph-mon[24787]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:11.243 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:11 smithi187 ceph-mon[27499]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:11.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:11 smithi159 ceph-mon[24787]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:13.245 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:13 smithi187 ceph-mon[27499]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:13.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:13 smithi159 ceph-mon[24787]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:14.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:14 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-30T13:21:14.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:14 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:14.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:14 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-30T13:21:14.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:14 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:15.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:15 smithi159 ceph-mon[24787]: Deploying daemon osd.6 on smithi187 2024-03-30T13:21:15.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:15 smithi159 ceph-mon[24787]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:15.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:15 smithi187 ceph-mon[27499]: Deploying daemon osd.6 on smithi187 2024-03-30T13:21:15.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:15 smithi187 ceph-mon[27499]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:17.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:17 smithi159 ceph-mon[24787]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:17.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:17 smithi187 ceph-mon[27499]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:19.438 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:19 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:19.438 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:19 smithi187 ceph-mon[27499]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:19.438 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:19 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:19.438 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:19 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:19.438 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:19 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:19.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:19 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:19.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:19 smithi159 ceph-mon[24787]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:19.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:19 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:19.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:19 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:19.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:19 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:21.017 INFO:teuthology.orchestra.run.smithi187.stdout:Created osd(s) 6 on host 'smithi187' 2024-03-30T13:21:21.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:21 smithi187 ceph-mon[27499]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:21.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:21 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:21.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:21 smithi159 ceph-mon[24787]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:21.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:21 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:21.627 DEBUG:teuthology.orchestra.run.smithi187:osd.6> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.6.service 2024-03-30T13:21:21.629 INFO:tasks.cephadm:Deploying osd.7 on smithi187 with /dev/vg_nvme/lv_1... 2024-03-30T13:21:21.629 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-30T13:21:22.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:22 smithi187 ceph-mon[27499]: from='osd.6 [v2:172.21.15.187:6816/4216578448,v1:172.21.15.187:6817/4216578448]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-30T13:21:22.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:22 smithi187 ceph-mon[27499]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-30T13:21:22.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:22 smithi159 ceph-mon[24787]: from='osd.6 [v2:172.21.15.187:6816/4216578448,v1:172.21.15.187:6817/4216578448]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-30T13:21:22.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:22 smithi159 ceph-mon[24787]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-30T13:21:23.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:23 smithi159 ceph-mon[24787]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:23.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:23 smithi159 ceph-mon[24787]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-30T13:21:23.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:23 smithi159 ceph-mon[24787]: osdmap e46: 7 total, 6 up, 7 in 2024-03-30T13:21:23.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:23.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:23 smithi159 ceph-mon[24787]: from='osd.6 [v2:172.21.15.187:6816/4216578448,v1:172.21.15.187:6817/4216578448]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:23.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:23 smithi159 ceph-mon[24787]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:23.505 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:23 smithi187 ceph-mon[27499]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:23.505 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:23 smithi187 ceph-mon[27499]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-30T13:21:23.505 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:23 smithi187 ceph-mon[27499]: osdmap e46: 7 total, 6 up, 7 in 2024-03-30T13:21:23.505 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:23.505 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:23 smithi187 ceph-mon[27499]: from='osd.6 [v2:172.21.15.187:6816/4216578448,v1:172.21.15.187:6817/4216578448]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:23.505 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:23 smithi187 ceph-mon[27499]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:23.505 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:21:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[37719]: 2024-03-30T13:21:23.176+0000 7f848fa27700 -1 osd.6 0 waiting for initial osdmap 2024-03-30T13:21:23.505 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:21:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[37719]: 2024-03-30T13:21:23.182+0000 7f848a3be700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:21:24.160 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-30T13:21:24.160 INFO:teuthology.orchestra.run.smithi187.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-30T13:21:24.160 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10+0 records in 2024-03-30T13:21:24.160 INFO:teuthology.orchestra.run.smithi187.stderr:10+0 records out 2024-03-30T13:21:24.160 INFO:teuthology.orchestra.run.smithi187.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0141266 s, 742 MB/s 2024-03-30T13:21:24.160 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping successful for: 2024-03-30T13:21:24.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:24 smithi187 ceph-mon[27499]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-30T13:21:24.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:24 smithi187 ceph-mon[27499]: osdmap e47: 7 total, 6 up, 7 in 2024-03-30T13:21:24.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:24 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:24.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:24 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:24.498 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:24 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:24.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:24 smithi159 ceph-mon[24787]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-30T13:21:24.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:24 smithi159 ceph-mon[24787]: osdmap e47: 7 total, 6 up, 7 in 2024-03-30T13:21:24.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:24 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:24.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:24 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:24.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:24 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:24.829 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi187:vg_nvme/lv_1 2024-03-30T13:21:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:25 smithi159 ceph-mon[24787]: purged_snaps scrub starts 2024-03-30T13:21:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:25 smithi159 ceph-mon[24787]: purged_snaps scrub ok 2024-03-30T13:21:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:25 smithi159 ceph-mon[24787]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:25 smithi159 ceph-mon[24787]: osd.6 [v2:172.21.15.187:6816/4216578448,v1:172.21.15.187:6817/4216578448] boot 2024-03-30T13:21:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:25 smithi159 ceph-mon[24787]: osdmap e48: 7 total, 7 up, 7 in 2024-03-30T13:21:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:25 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:25 smithi159 ceph-mon[24787]: osdmap e49: 7 total, 7 up, 7 in 2024-03-30T13:21:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:25 smithi187 ceph-mon[27499]: purged_snaps scrub starts 2024-03-30T13:21:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:25 smithi187 ceph-mon[27499]: purged_snaps scrub ok 2024-03-30T13:21:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:25 smithi187 ceph-mon[27499]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-30T13:21:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:25 smithi187 ceph-mon[27499]: osd.6 [v2:172.21.15.187:6816/4216578448,v1:172.21.15.187:6817/4216578448] boot 2024-03-30T13:21:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:25 smithi187 ceph-mon[27499]: osdmap e48: 7 total, 7 up, 7 in 2024-03-30T13:21:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:25 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:21:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:25 smithi187 ceph-mon[27499]: osdmap e49: 7 total, 7 up, 7 in 2024-03-30T13:21:27.435 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:27 smithi187 ceph-mon[27499]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:27.435 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:27 smithi187 ceph-mon[27499]: osdmap e50: 7 total, 7 up, 7 in 2024-03-30T13:21:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:27 smithi159 ceph-mon[24787]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:27 smithi159 ceph-mon[24787]: osdmap e50: 7 total, 7 up, 7 in 2024-03-30T13:21:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: osdmap e51: 7 total, 7 up, 7 in 2024-03-30T13:21:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:28.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:28.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:28.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:28.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:28.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:28 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:28.506 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: osdmap e51: 7 total, 7 up, 7 in 2024-03-30T13:21:28.506 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:28.506 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:28.506 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:28.506 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:28.507 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:28.507 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:28.507 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:28.507 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:28.507 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:28.507 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:28 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:29 smithi159 ceph-mon[24787]: Detected new or changed devices on smithi187 2024-03-30T13:21:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:29 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi187 to 5312M 2024-03-30T13:21:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:29 smithi159 ceph-mon[24787]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:29 smithi159 ceph-mon[24787]: osdmap e52: 7 total, 7 up, 7 in 2024-03-30T13:21:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:29 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:21:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:29 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:21:29.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:29 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:29 smithi187 ceph-mon[27499]: Detected new or changed devices on smithi187 2024-03-30T13:21:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:29 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi187 to 5312M 2024-03-30T13:21:29.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:29 smithi187 ceph-mon[27499]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:29.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:29 smithi187 ceph-mon[27499]: osdmap e52: 7 total, 7 up, 7 in 2024-03-30T13:21:29.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:29 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-30T13:21:29.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:29 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-30T13:21:29.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:29 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:30.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:30 smithi159 ceph-mon[24787]: from='client.24207 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:21:30.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:30 smithi187 ceph-mon[27499]: from='client.24207 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:21:31.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:31 smithi159 ceph-mon[24787]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:31.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:31 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/3333626970' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a6f07ed-5b20-453b-bce0-fb1a06061fea"}]: dispatch 2024-03-30T13:21:31.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:31 smithi159 ceph-mon[24787]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a6f07ed-5b20-453b-bce0-fb1a06061fea"}]: dispatch 2024-03-30T13:21:31.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:31 smithi159 ceph-mon[24787]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8a6f07ed-5b20-453b-bce0-fb1a06061fea"}]': finished 2024-03-30T13:21:31.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:31 smithi159 ceph-mon[24787]: osdmap e53: 8 total, 7 up, 8 in 2024-03-30T13:21:31.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:31 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:31.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:31 smithi187 ceph-mon[27499]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:31.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:31 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/3333626970' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a6f07ed-5b20-453b-bce0-fb1a06061fea"}]: dispatch 2024-03-30T13:21:31.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:31 smithi187 ceph-mon[27499]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a6f07ed-5b20-453b-bce0-fb1a06061fea"}]: dispatch 2024-03-30T13:21:31.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:31 smithi187 ceph-mon[27499]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8a6f07ed-5b20-453b-bce0-fb1a06061fea"}]': finished 2024-03-30T13:21:31.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:31 smithi187 ceph-mon[27499]: osdmap e53: 8 total, 7 up, 8 in 2024-03-30T13:21:31.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:31 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:32.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:32 smithi159 ceph-mon[24787]: from='client.? 172.21.15.187:0/3911310730' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:21:32.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:32 smithi187 ceph-mon[27499]: from='client.? 172.21.15.187:0/3911310730' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-30T13:21:33.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:33 smithi159 ceph-mon[24787]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-03-30T13:21:33.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:33 smithi187 ceph-mon[27499]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-03-30T13:21:34.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:34 smithi159 ceph-mon[24787]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-03-30T13:21:34.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:34 smithi187 ceph-mon[27499]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-03-30T13:21:37.302 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:37 smithi187 ceph-mon[27499]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-30T13:21:37.302 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-30T13:21:37.302 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:37 smithi159 ceph-mon[24787]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-30T13:21:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-30T13:21:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:38.372 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:38 smithi187 ceph-mon[27499]: Deploying daemon osd.7 on smithi187 2024-03-30T13:21:38.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:38 smithi159 ceph-mon[24787]: Deploying daemon osd.7 on smithi187 2024-03-30T13:21:39.479 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:39 smithi187 ceph-mon[27499]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-30T13:21:39.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:39 smithi159 ceph-mon[24787]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-30T13:21:41.462 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:41 smithi187 ceph-mon[27499]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-30T13:21:41.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:41 smithi159 ceph-mon[24787]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-30T13:21:43.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:43.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:43.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:43.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:43.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:42 smithi159 ceph-mon[24787]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:43.009 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:43.009 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:43.009 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:43.009 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:43.009 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:42 smithi187 ceph-mon[27499]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:44.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:44 smithi159 ceph-mon[24787]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:44.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:44 smithi187 ceph-mon[27499]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:45.450 INFO:teuthology.orchestra.run.smithi187.stdout:Created osd(s) 7 on host 'smithi187' 2024-03-30T13:21:45.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:45 smithi159 ceph-mon[24787]: from='osd.7 [v2:172.21.15.187:6824/1091192097,v1:172.21.15.187:6825/1091192097]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-30T13:21:45.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:45 smithi159 ceph-mon[24787]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-30T13:21:45.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:45 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:45.756 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:45 smithi187 ceph-mon[27499]: from='osd.7 [v2:172.21.15.187:6824/1091192097,v1:172.21.15.187:6825/1091192097]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-30T13:21:45.756 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:45 smithi187 ceph-mon[27499]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-30T13:21:45.756 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:45 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:46.133 DEBUG:teuthology.orchestra.run.smithi187:osd.7> sudo journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.7.service 2024-03-30T13:21:46.136 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-30T13:21:46.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-03-30T13:21:46.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-30T13:21:46.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: osdmap e54: 8 total, 7 up, 8 in 2024-03-30T13:21:46.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:46.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: from='osd.7 [v2:172.21.15.187:6824/1091192097,v1:172.21.15.187:6825/1091192097]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:46.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:46.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:46.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:46.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:21:46.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: osdmap e54: 8 total, 7 up, 8 in 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: from='osd.7 [v2:172.21.15.187:6824/1091192097,v1:172.21.15.187:6825/1091192097]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:21:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:21:46.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[40797]: 2024-03-30T13:21:46.491+0000 7fc837a5e700 -1 osd.7 0 waiting for initial osdmap 2024-03-30T13:21:46.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:21:46 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[40797]: 2024-03-30T13:21:46.499+0000 7fc8313f3700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:21:47.602 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:47 smithi187 ceph-mon[27499]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-30T13:21:47.603 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:47 smithi187 ceph-mon[27499]: osdmap e55: 8 total, 7 up, 8 in 2024-03-30T13:21:47.603 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:47.603 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:47.720 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:47 smithi159 ceph-mon[24787]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-30T13:21:47.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:47 smithi159 ceph-mon[24787]: osdmap e55: 8 total, 7 up, 8 in 2024-03-30T13:21:47.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:47.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:48.126 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: purged_snaps scrub starts 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: purged_snaps scrub ok 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: osd.7 [v2:172.21.15.187:6824/1091192097,v1:172.21.15.187:6825/1091192097] boot 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: osdmap e56: 8 total, 8 up, 8 in 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3503788912' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-30T13:21:48.682 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:48 smithi159 ceph-mon[24787]: osdmap e57: 8 total, 8 up, 8 in 2024-03-30T13:21:48.683 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1711804907,"num_in_osds":8,"osd_in_since":1711804890,"num_remapped_pgs":1} 2024-03-30T13:21:48.684 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-03-30T13:21:48.756 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: purged_snaps scrub starts 2024-03-30T13:21:48.756 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: purged_snaps scrub ok 2024-03-30T13:21:48.756 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:48.756 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: osd.7 [v2:172.21.15.187:6824/1091192097,v1:172.21.15.187:6825/1091192097] boot 2024-03-30T13:21:48.756 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: osdmap e56: 8 total, 8 up, 8 in 2024-03-30T13:21:48.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:21:48.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-30T13:21:48.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3503788912' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-30T13:21:48.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:48 smithi187 ceph-mon[27499]: osdmap e57: 8 total, 8 up, 8 in 2024-03-30T13:21:48.917 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:21:50.000 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: Detected new or changed devices on smithi187 2024-03-30T13:21:50.000 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:50.000 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:50.000 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:50.000 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:50.000 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:50.000 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi187 to 3984M 2024-03-30T13:21:50.000 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:50.001 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:50.001 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:50.001 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:50.001 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:50.001 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:50.001 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:49 smithi159 ceph-mon[24787]: osdmap e58: 8 total, 8 up, 8 in 2024-03-30T13:21:50.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: Detected new or changed devices on smithi187 2024-03-30T13:21:50.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:50.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:50.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:50.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:50.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:21:50.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi187 to 3984M 2024-03-30T13:21:50.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:50.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:50.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:21:50.275 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:21:50.275 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:21:50.275 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:21:50.275 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:49 smithi187 ceph-mon[27499]: osdmap e58: 8 total, 8 up, 8 in 2024-03-30T13:21:50.642 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:21:50.643 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":59,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","created":"2024-03-30T13:15:53.970627+0000","modified":"2024-03-30T13:21:50.474125+0000","last_up_change":"2024-03-30T13:21:47.488238+0000","last_in_change":"2024-03-30T13:21:30.707829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-30T13:19:50.393013+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"15f9bf93-f28e-4ef1-a9f2-37a4ed2b0296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6803","nonce":1641385808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6805","nonce":1641385808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6809","nonce":1641385808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6807","nonce":1641385808}]},"public_addr":"172.21.15.159:6803/1641385808","cluster_addr":"172.21.15.159:6805/1641385808","heartbeat_back_addr":"172.21.15.159:6809/1641385808","heartbeat_front_addr":"172.21.15.159:6807/1641385808","state":["exists","up"]},{"osd":1,"uuid":"f5e3306c-4090-412b-ab16-26224afae769","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6811","nonce":2880408563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6813","nonce":2880408563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6817","nonce":2880408563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6815","nonce":2880408563}]},"public_addr":"172.21.15.159:6811/2880408563","cluster_addr":"172.21.15.159:6813/2880408563","heartbeat_back_addr":"172.21.15.159:6817/2880408563","heartbeat_front_addr":"172.21.15.159:6815/2880408563","state":["exists","up"]},{"osd":2,"uuid":"6ddd12f2-b128-4871-b107-4cc27d6453ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6819","nonce":1039946591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6821","nonce":1039946591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6825","nonce":1039946591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6823","nonce":1039946591}]},"public_addr":"172.21.15.159:6819/1039946591","cluster_addr":"172.21.15.159:6821/1039946591","heartbeat_back_addr":"172.21.15.159:6825/1039946591","heartbeat_front_addr":"172.21.15.159:6823/1039946591","state":["exists","up"]},{"osd":3,"uuid":"1e990270-44f5-4f81-b3fc-5ffce2cfd21f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6827","nonce":1390459583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6829","nonce":1390459583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6833","nonce":1390459583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6831","nonce":1390459583}]},"public_addr":"172.21.15.159:6827/1390459583","cluster_addr":"172.21.15.159:6829/1390459583","heartbeat_back_addr":"172.21.15.159:6833/1390459583","heartbeat_front_addr":"172.21.15.159:6831/1390459583","state":["exists","up"]},{"osd":4,"uuid":"ff5b3592-6def-4081-8e9c-4d89e0607324","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6801","nonce":2785191302}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6803","nonce":2785191302}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6807","nonce":2785191302}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6805","nonce":2785191302}]},"public_addr":"172.21.15.187:6801/2785191302","cluster_addr":"172.21.15.187:6803/2785191302","heartbeat_back_addr":"172.21.15.187:6807/2785191302","heartbeat_front_addr":"172.21.15.187:6805/2785191302","state":["exists","up"]},{"osd":5,"uuid":"f72f07bc-6453-4ff6-b0bf-a6f9062d5696","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6809","nonce":1659074308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6811","nonce":1659074308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6815","nonce":1659074308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6813","nonce":1659074308}]},"public_addr":"172.21.15.187:6809/1659074308","cluster_addr":"172.21.15.187:6811/1659074308","heartbeat_back_addr":"172.21.15.187:6815/1659074308","heartbeat_front_addr":"172.21.15.187:6813/1659074308","state":["exists","up"]},{"osd":6,"uuid":"17b28af4-348e-4983-9c5e-d19fb58e0624","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6817","nonce":4216578448}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6819","nonce":4216578448}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6823","nonce":4216578448}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6821","nonce":4216578448}]},"public_addr":"172.21.15.187:6817/4216578448","cluster_addr":"172.21.15.187:6819/4216578448","heartbeat_back_addr":"172.21.15.187:6823/4216578448","heartbeat_front_addr":"172.21.15.187:6821/4216578448","state":["exists","up"]},{"osd":7,"uuid":"8a6f07ed-5b20-453b-bce0-fb1a06061fea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6825","nonce":1091192097}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6827","nonce":1091192097}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6831","nonce":1091192097}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6829","nonce":1091192097}]},"public_addr":"172.21.15.187:6825/1091192097","cluster_addr":"172.21.15.187:6827/1091192097","heartbeat_back_addr":"172.21.15.187:6831/1091192097","heartbeat_front_addr":"172.21.15.187:6829/1091192097","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:18:51.766121+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:19:18.569717+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:19:46.373110+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:15.258680+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:36.470121+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:59.540035+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:21:22.207572+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:21:45.651859+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/1799940320":"2024-03-31T13:17:46.017751+0000","172.21.15.159:6800/2739033964":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/913896798":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/965034129":"2024-03-31T13:16:50.550394+0000","172.21.15.159:0/2718272540":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/1604279307":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/2739033964":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/1016585273":"2024-03-31T13:16:22.685809+0000","172.21.15.159:6800/1604279307":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/4180582238":"2024-03-31T13:16:22.685809+0000","172.21.15.159:0/607465004":"2024-03-31T13:16:22.685809+0000","172.21.15.159:6800/4180582238":"2024-03-31T13:16:22.685809+0000","172.21.15.159:0/1272528917":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/753599827":"2024-03-31T13:16:50.550394+0000","172.21.15.159:0/4070436025":"2024-03-31T13:16:22.685809+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-30T13:21:50.998 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:50 smithi159 ceph-mon[24787]: pgmap v168: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-30T13:21:50.998 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:50 smithi159 ceph-mon[24787]: osdmap e59: 8 total, 8 up, 8 in 2024-03-30T13:21:50.998 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:50 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3605853270' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-30T13:21:51.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:50 smithi187 ceph-mon[27499]: pgmap v168: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-30T13:21:51.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:50 smithi187 ceph-mon[27499]: osdmap e59: 8 total, 8 up, 8 in 2024-03-30T13:21:51.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:50 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3605853270' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-30T13:21:51.412 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-30T13:19:50.393013+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-03-30T13:21:51.412 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-03-30T13:21:51.635 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:21:52.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:52 smithi159 ceph-mon[24787]: osdmap e60: 8 total, 8 up, 8 in 2024-03-30T13:21:52.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:52 smithi159 ceph-mon[24787]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-30T13:21:52.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:52 smithi187 ceph-mon[27499]: osdmap e60: 8 total, 8 up, 8 in 2024-03-30T13:21:52.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:52 smithi187 ceph-mon[27499]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-30T13:21:53.403 INFO:teuthology.orchestra.run.smithi159.stdout:pg_num: 1 2024-03-30T13:21:53.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:53 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/851778372' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-30T13:21:53.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:53 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/851778372' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-30T13:21:53.979 INFO:tasks.cephadm:Setting up client nodes... 2024-03-30T13:21:53.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-30T13:21:54.535 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:54 smithi159 ceph-mon[24787]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-03-30T13:21:54.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:54 smithi187 ceph-mon[27499]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-03-30T13:21:56.049 INFO:teuthology.orchestra.run.smithi159.stdout:[client.0] 2024-03-30T13:21:56.049 INFO:teuthology.orchestra.run.smithi159.stdout: key = AQD0EQhmED2lAhAA4gQSBkeScKqhaevma3MOvg== 2024-03-30T13:21:56.374 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:56 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3113538311' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-30T13:21:56.375 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:56 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3113538311' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-30T13:21:56.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:56 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3113538311' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-30T13:21:56.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:56 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3113538311' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-30T13:21:56.756 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-03-30T13:21:56.756 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-30T13:21:56.756 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-30T13:21:56.798 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-30T13:21:56.798 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-30T13:21:56.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph mgr dump --format=json 2024-03-30T13:21:57.051 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:21:57.461 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:57 smithi159 ceph-mon[24787]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-03-30T13:21:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:57 smithi187 ceph-mon[27499]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-03-30T13:21:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:21:59.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:59 smithi159 ceph-mon[24787]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:21:59.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:21:59 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3150018628' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-30T13:21:59.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:59 smithi187 ceph-mon[27499]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:21:59.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:21:59 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3150018628' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-30T13:21:59.653 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi159.fnasly","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":1132848898},{"type":"v1","addr":"172.21.15.159:6801","nonce":1132848898}]},"active_addr":"172.21.15.159:6801/1132848898","active_change":"2024-03-30T13:17:46.018053+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi187.jpkaud","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.159:8443/","prometheus":"http://172.21.15.159:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.159:0","nonce":1156397387}]},{"addrvec":[{"type":"v2","addr":"172.21.15.159:0","nonce":3756296704}]},{"addrvec":[{"type":"v2","addr":"172.21.15.159:0","nonce":3234489602}]},{"addrvec":[{"type":"v2","addr":"172.21.15.159:0","nonce":3626917998}]}]}} 2024-03-30T13:21:59.658 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-30T13:21:59.658 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-30T13:21:59.658 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-03-30T13:21:59.876 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:01.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:01 smithi159 ceph-mon[24787]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:01 smithi187 ceph-mon[27499]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:01.641 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:22:01.642 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":60,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","created":"2024-03-30T13:15:53.970627+0000","modified":"2024-03-30T13:21:51.475434+0000","last_up_change":"2024-03-30T13:21:47.488238+0000","last_in_change":"2024-03-30T13:21:30.707829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-30T13:19:50.393013+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"15f9bf93-f28e-4ef1-a9f2-37a4ed2b0296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6803","nonce":1641385808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6805","nonce":1641385808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6809","nonce":1641385808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6807","nonce":1641385808}]},"public_addr":"172.21.15.159:6803/1641385808","cluster_addr":"172.21.15.159:6805/1641385808","heartbeat_back_addr":"172.21.15.159:6809/1641385808","heartbeat_front_addr":"172.21.15.159:6807/1641385808","state":["exists","up"]},{"osd":1,"uuid":"f5e3306c-4090-412b-ab16-26224afae769","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6811","nonce":2880408563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6813","nonce":2880408563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6817","nonce":2880408563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6815","nonce":2880408563}]},"public_addr":"172.21.15.159:6811/2880408563","cluster_addr":"172.21.15.159:6813/2880408563","heartbeat_back_addr":"172.21.15.159:6817/2880408563","heartbeat_front_addr":"172.21.15.159:6815/2880408563","state":["exists","up"]},{"osd":2,"uuid":"6ddd12f2-b128-4871-b107-4cc27d6453ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6819","nonce":1039946591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6821","nonce":1039946591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6825","nonce":1039946591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6823","nonce":1039946591}]},"public_addr":"172.21.15.159:6819/1039946591","cluster_addr":"172.21.15.159:6821/1039946591","heartbeat_back_addr":"172.21.15.159:6825/1039946591","heartbeat_front_addr":"172.21.15.159:6823/1039946591","state":["exists","up"]},{"osd":3,"uuid":"1e990270-44f5-4f81-b3fc-5ffce2cfd21f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6827","nonce":1390459583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6829","nonce":1390459583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6833","nonce":1390459583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6831","nonce":1390459583}]},"public_addr":"172.21.15.159:6827/1390459583","cluster_addr":"172.21.15.159:6829/1390459583","heartbeat_back_addr":"172.21.15.159:6833/1390459583","heartbeat_front_addr":"172.21.15.159:6831/1390459583","state":["exists","up"]},{"osd":4,"uuid":"ff5b3592-6def-4081-8e9c-4d89e0607324","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6801","nonce":2785191302}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6803","nonce":2785191302}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6807","nonce":2785191302}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6805","nonce":2785191302}]},"public_addr":"172.21.15.187:6801/2785191302","cluster_addr":"172.21.15.187:6803/2785191302","heartbeat_back_addr":"172.21.15.187:6807/2785191302","heartbeat_front_addr":"172.21.15.187:6805/2785191302","state":["exists","up"]},{"osd":5,"uuid":"f72f07bc-6453-4ff6-b0bf-a6f9062d5696","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6809","nonce":1659074308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6811","nonce":1659074308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6815","nonce":1659074308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6813","nonce":1659074308}]},"public_addr":"172.21.15.187:6809/1659074308","cluster_addr":"172.21.15.187:6811/1659074308","heartbeat_back_addr":"172.21.15.187:6815/1659074308","heartbeat_front_addr":"172.21.15.187:6813/1659074308","state":["exists","up"]},{"osd":6,"uuid":"17b28af4-348e-4983-9c5e-d19fb58e0624","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6817","nonce":4216578448}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6819","nonce":4216578448}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6823","nonce":4216578448}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6821","nonce":4216578448}]},"public_addr":"172.21.15.187:6817/4216578448","cluster_addr":"172.21.15.187:6819/4216578448","heartbeat_back_addr":"172.21.15.187:6823/4216578448","heartbeat_front_addr":"172.21.15.187:6821/4216578448","state":["exists","up"]},{"osd":7,"uuid":"8a6f07ed-5b20-453b-bce0-fb1a06061fea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6825","nonce":1091192097}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6827","nonce":1091192097}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6831","nonce":1091192097}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6829","nonce":1091192097}]},"public_addr":"172.21.15.187:6825/1091192097","cluster_addr":"172.21.15.187:6827/1091192097","heartbeat_back_addr":"172.21.15.187:6831/1091192097","heartbeat_front_addr":"172.21.15.187:6829/1091192097","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:18:51.766121+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:19:18.569717+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:19:46.373110+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:15.258680+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:36.470121+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:59.540035+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:21:22.207572+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:21:45.651859+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/1799940320":"2024-03-31T13:17:46.017751+0000","172.21.15.159:6800/2739033964":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/913896798":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/965034129":"2024-03-31T13:16:50.550394+0000","172.21.15.159:0/2718272540":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/1604279307":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/2739033964":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/1016585273":"2024-03-31T13:16:22.685809+0000","172.21.15.159:6800/1604279307":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/4180582238":"2024-03-31T13:16:22.685809+0000","172.21.15.159:0/607465004":"2024-03-31T13:16:22.685809+0000","172.21.15.159:6800/4180582238":"2024-03-31T13:16:22.685809+0000","172.21.15.159:0/1272528917":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/753599827":"2024-03-31T13:16:50.550394+0000","172.21.15.159:0/4070436025":"2024-03-31T13:16:22.685809+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-30T13:22:02.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:02 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3777161621' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-30T13:22:02.405 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-30T13:22:02.405 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-03-30T13:22:02.489 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:02 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3777161621' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-30T13:22:02.625 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:03.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:03 smithi159 ceph-mon[24787]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:03 smithi187 ceph-mon[27499]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:04.428 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:22:04.428 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":60,"fsid":"9058771a-ee97-11ee-b647-cb9ed24678a4","created":"2024-03-30T13:15:53.970627+0000","modified":"2024-03-30T13:21:51.475434+0000","last_up_change":"2024-03-30T13:21:47.488238+0000","last_in_change":"2024-03-30T13:21:30.707829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-30T13:19:50.393013+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"15f9bf93-f28e-4ef1-a9f2-37a4ed2b0296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6803","nonce":1641385808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6805","nonce":1641385808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6809","nonce":1641385808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":1641385808},{"type":"v1","addr":"172.21.15.159:6807","nonce":1641385808}]},"public_addr":"172.21.15.159:6803/1641385808","cluster_addr":"172.21.15.159:6805/1641385808","heartbeat_back_addr":"172.21.15.159:6809/1641385808","heartbeat_front_addr":"172.21.15.159:6807/1641385808","state":["exists","up"]},{"osd":1,"uuid":"f5e3306c-4090-412b-ab16-26224afae769","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6811","nonce":2880408563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6813","nonce":2880408563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6817","nonce":2880408563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":2880408563},{"type":"v1","addr":"172.21.15.159:6815","nonce":2880408563}]},"public_addr":"172.21.15.159:6811/2880408563","cluster_addr":"172.21.15.159:6813/2880408563","heartbeat_back_addr":"172.21.15.159:6817/2880408563","heartbeat_front_addr":"172.21.15.159:6815/2880408563","state":["exists","up"]},{"osd":2,"uuid":"6ddd12f2-b128-4871-b107-4cc27d6453ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6819","nonce":1039946591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6821","nonce":1039946591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6825","nonce":1039946591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1039946591},{"type":"v1","addr":"172.21.15.159:6823","nonce":1039946591}]},"public_addr":"172.21.15.159:6819/1039946591","cluster_addr":"172.21.15.159:6821/1039946591","heartbeat_back_addr":"172.21.15.159:6825/1039946591","heartbeat_front_addr":"172.21.15.159:6823/1039946591","state":["exists","up"]},{"osd":3,"uuid":"1e990270-44f5-4f81-b3fc-5ffce2cfd21f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6827","nonce":1390459583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6829","nonce":1390459583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6833","nonce":1390459583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":1390459583},{"type":"v1","addr":"172.21.15.159:6831","nonce":1390459583}]},"public_addr":"172.21.15.159:6827/1390459583","cluster_addr":"172.21.15.159:6829/1390459583","heartbeat_back_addr":"172.21.15.159:6833/1390459583","heartbeat_front_addr":"172.21.15.159:6831/1390459583","state":["exists","up"]},{"osd":4,"uuid":"ff5b3592-6def-4081-8e9c-4d89e0607324","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6801","nonce":2785191302}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6803","nonce":2785191302}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6807","nonce":2785191302}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":2785191302},{"type":"v1","addr":"172.21.15.187:6805","nonce":2785191302}]},"public_addr":"172.21.15.187:6801/2785191302","cluster_addr":"172.21.15.187:6803/2785191302","heartbeat_back_addr":"172.21.15.187:6807/2785191302","heartbeat_front_addr":"172.21.15.187:6805/2785191302","state":["exists","up"]},{"osd":5,"uuid":"f72f07bc-6453-4ff6-b0bf-a6f9062d5696","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6809","nonce":1659074308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6811","nonce":1659074308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6815","nonce":1659074308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":1659074308},{"type":"v1","addr":"172.21.15.187:6813","nonce":1659074308}]},"public_addr":"172.21.15.187:6809/1659074308","cluster_addr":"172.21.15.187:6811/1659074308","heartbeat_back_addr":"172.21.15.187:6815/1659074308","heartbeat_front_addr":"172.21.15.187:6813/1659074308","state":["exists","up"]},{"osd":6,"uuid":"17b28af4-348e-4983-9c5e-d19fb58e0624","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6817","nonce":4216578448}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6819","nonce":4216578448}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6823","nonce":4216578448}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":4216578448},{"type":"v1","addr":"172.21.15.187:6821","nonce":4216578448}]},"public_addr":"172.21.15.187:6817/4216578448","cluster_addr":"172.21.15.187:6819/4216578448","heartbeat_back_addr":"172.21.15.187:6823/4216578448","heartbeat_front_addr":"172.21.15.187:6821/4216578448","state":["exists","up"]},{"osd":7,"uuid":"8a6f07ed-5b20-453b-bce0-fb1a06061fea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6825","nonce":1091192097}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6827","nonce":1091192097}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6831","nonce":1091192097}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":1091192097},{"type":"v1","addr":"172.21.15.187:6829","nonce":1091192097}]},"public_addr":"172.21.15.187:6825/1091192097","cluster_addr":"172.21.15.187:6827/1091192097","heartbeat_back_addr":"172.21.15.187:6831/1091192097","heartbeat_front_addr":"172.21.15.187:6829/1091192097","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:18:51.766121+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:19:18.569717+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:19:46.373110+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:15.258680+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:36.470121+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:20:59.540035+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:21:22.207572+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-30T13:21:45.651859+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/1799940320":"2024-03-31T13:17:46.017751+0000","172.21.15.159:6800/2739033964":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/913896798":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/965034129":"2024-03-31T13:16:50.550394+0000","172.21.15.159:0/2718272540":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/1604279307":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/2739033964":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/1016585273":"2024-03-31T13:16:22.685809+0000","172.21.15.159:6800/1604279307":"2024-03-31T13:16:50.550394+0000","172.21.15.159:6801/4180582238":"2024-03-31T13:16:22.685809+0000","172.21.15.159:0/607465004":"2024-03-31T13:16:22.685809+0000","172.21.15.159:6800/4180582238":"2024-03-31T13:16:22.685809+0000","172.21.15.159:0/1272528917":"2024-03-31T13:17:46.017751+0000","172.21.15.159:0/753599827":"2024-03-31T13:16:50.550394+0000","172.21.15.159:0/4070436025":"2024-03-31T13:16:22.685809+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-30T13:22:04.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:04 smithi159 ceph-mon[24787]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:04.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:04 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3850253521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-30T13:22:04.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:04 smithi187 ceph-mon[27499]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:04.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:04 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3850253521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-30T13:22:05.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-03-30T13:22:05.195 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-03-30T13:22:05.195 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-03-30T13:22:05.196 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-03-30T13:22:05.196 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-03-30T13:22:05.196 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-03-30T13:22:05.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-03-30T13:22:05.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-03-30T13:22:05.637 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:05.638 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:05.638 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:05.639 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:05.641 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:05.641 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:05.642 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:05.645 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:07.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:07 smithi159 ceph-mon[24787]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:07.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:07 smithi187 ceph-mon[27499]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:09.394 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:09 smithi159 ceph-mon[24787]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:09 smithi187 ceph-mon[27499]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:11.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:11 smithi159 ceph-mon[24787]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:11.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:11 smithi187 ceph-mon[27499]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:13.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:13 smithi159 ceph-mon[24787]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:13.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:13 smithi187 ceph-mon[27499]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:14.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:14 smithi159 ceph-mon[24787]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:14.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:14 smithi187 ceph-mon[27499]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:17.327 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:17 smithi159 ceph-mon[24787]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:17.355 INFO:teuthology.orchestra.run.smithi159.stdout:60129542182 2024-03-30T13:22:17.356 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-03-30T13:22:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:17 smithi187 ceph-mon[27499]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:18.348 INFO:teuthology.orchestra.run.smithi159.stdout:240518168585 2024-03-30T13:22:18.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-03-30T13:22:18.977 INFO:teuthology.orchestra.run.smithi159.stdout:115964117019 2024-03-30T13:22:18.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-03-30T13:22:19.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:19 smithi159 ceph-mon[24787]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:19 smithi187 ceph-mon[27499]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:19.641 INFO:teuthology.orchestra.run.smithi159.stdout:206158430221 2024-03-30T13:22:19.641 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-03-30T13:22:20.727 INFO:teuthology.orchestra.run.smithi159.stdout:81604378657 2024-03-30T13:22:20.728 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-03-30T13:22:20.755 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:21.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:21 smithi159 ceph-mon[24787]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:21.377 INFO:teuthology.orchestra.run.smithi159.stdout:38654705708 2024-03-30T13:22:21.378 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-03-30T13:22:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:21 smithi187 ceph-mon[27499]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:22.646 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:22.648 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:22.650 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:22.650 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:23.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:23 smithi159 ceph-mon[24787]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:23.439 INFO:teuthology.orchestra.run.smithi159.stdout:137438953496 2024-03-30T13:22:23.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-03-30T13:22:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:23 smithi187 ceph-mon[27499]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:24.748 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:24.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:24 smithi159 ceph-mon[24787]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:24 smithi187 ceph-mon[27499]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:27.233 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:27 smithi159 ceph-mon[24787]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:27 smithi187 ceph-mon[27499]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:27.616 INFO:teuthology.orchestra.run.smithi159.stdout:240518168587 2024-03-30T13:22:27.946 INFO:teuthology.orchestra.run.smithi159.stdout:171798691859 2024-03-30T13:22:27.946 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-03-30T13:22:28.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:28 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3187615290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-30T13:22:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:28 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3187615290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-30T13:22:29.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:29 smithi159 ceph-mon[24787]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:29 smithi187 ceph-mon[27499]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:29.592 INFO:teuthology.orchestra.run.smithi159.stdout:60129542185 2024-03-30T13:22:29.821 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168587 for osd.7 2024-03-30T13:22:29.822 DEBUG:teuthology.parallel:result is None 2024-03-30T13:22:30.446 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:30 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/342406737' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-30T13:22:30.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:30 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/342406737' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-30T13:22:30.584 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542185 for osd.1 2024-03-30T13:22:30.584 DEBUG:teuthology.parallel:result is None 2024-03-30T13:22:30.956 INFO:teuthology.orchestra.run.smithi159.stdout:81604378660 2024-03-30T13:22:31.441 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:31 smithi187 ceph-mon[27499]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:31.441 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:31 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/904904563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-30T13:22:31.457 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:31.457 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:31.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:31 smithi159 ceph-mon[24787]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:31.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:31 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/904904563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-30T13:22:32.620 INFO:teuthology.orchestra.run.smithi159.stdout:115964117022 2024-03-30T13:22:32.833 INFO:teuthology.orchestra.run.smithi159.stdout:38654705711 2024-03-30T13:22:33.338 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117019 got 115964117022 for osd.3 2024-03-30T13:22:33.338 DEBUG:teuthology.parallel:result is None 2024-03-30T13:22:33.449 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:33 smithi159 ceph-mon[24787]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:33.450 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:33 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1400655624' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-30T13:22:33.450 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:33 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2857395866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-30T13:22:33.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:33 smithi187 ceph-mon[27499]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:33.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:33 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1400655624' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-30T13:22:33.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:33 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/2857395866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-30T13:22:33.776 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378660 for osd.2 2024-03-30T13:22:33.776 DEBUG:teuthology.parallel:result is None 2024-03-30T13:22:34.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:34 smithi159 ceph-mon[24787]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:34.619 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705711 for osd.0 2024-03-30T13:22:34.619 DEBUG:teuthology.parallel:result is None 2024-03-30T13:22:34.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:34 smithi187 ceph-mon[27499]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:35.274 INFO:teuthology.orchestra.run.smithi159.stdout:206158430225 2024-03-30T13:22:35.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:35 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/217525143' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-30T13:22:35.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:35 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/217525143' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-30T13:22:36.702 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:36 smithi159 ceph-mon[24787]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:36.731 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430221 got 206158430225 for osd.6 2024-03-30T13:22:36.731 DEBUG:teuthology.parallel:result is None 2024-03-30T13:22:36.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:36 smithi187 ceph-mon[27499]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:38.435 INFO:teuthology.orchestra.run.smithi159.stdout:137438953499 2024-03-30T13:22:38.667 INFO:teuthology.orchestra.run.smithi159.stdout:171798691863 2024-03-30T13:22:39.040 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953496 got 137438953499 for osd.4 2024-03-30T13:22:39.040 DEBUG:teuthology.parallel:result is None 2024-03-30T13:22:39.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:39 smithi159 ceph-mon[24787]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:39.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:39 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/2815695016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-30T13:22:39.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:39 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1483323193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-30T13:22:39.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:39 smithi187 ceph-mon[27499]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:39.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:39 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/2815695016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-30T13:22:39.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:39 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1483323193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-30T13:22:39.638 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691863 for osd.5 2024-03-30T13:22:39.638 DEBUG:teuthology.parallel:result is None 2024-03-30T13:22:39.638 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-30T13:22:39.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-03-30T13:22:39.864 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:41.333 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:41 smithi159 ceph-mon[24787]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:41.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:41 smithi187 ceph-mon[27499]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:41.720 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:22:41.722 INFO:teuthology.orchestra.run.smithi159.stderr:dumped all 2024-03-30T13:22:42.409 INFO:teuthology.orchestra.run.smithi159.stdout:{"pg_ready":true,"pg_map":{"version":195,"stamp":"2024-03-30T13:22:40.122678+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48936,"kb_used_data":6376,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912280,"statfs":{"total":767960285184,"available":767910174720,"internally_reserved":0,"allocated":6529024,"data_stored":3571320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002397"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-03-30T13:21:51.483087+0000","last_change":"2024-03-30T13:21:51.483087+0000","last_active":"2024-03-30T13:21:51.483087+0000","last_peered":"2024-03-30T13:21:51.483087+0000","last_clean":"2024-03-30T13:21:51.483087+0000","last_became_active":"2024-03-30T13:21:51.482618+0000","last_became_peered":"2024-03-30T13:21:51.482618+0000","last_unstale":"2024-03-30T13:21:51.483087+0000","last_undegraded":"2024-03-30T13:21:51.483087+0000","last_fullsized":"2024-03-30T13:21:51.483087+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-30T13:19:50.443163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-30T13:19:50.443163+0000","last_clean_scrub_stamp":"2024-03-30T13:19:50.443163+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-31T16:56:46.793610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53500000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.497}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34000000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]}]},{"osd":6,"up_from":48,"seq":206158430226,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":776554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.244,"5min":1.244,"15min":1.244},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":13.776999999999999,"5min":13.776999999999999,"15min":13.776999999999999},"last":0.45100000000000001}]},{"osd":1,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.234,"15min":1.234},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":13.792,"5min":13.792,"15min":13.792},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.58999999999999997}]},{"osd":2,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.52900000000000003}]},{"osd":3,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.626}]},{"osd":4,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.63700000000000001}]},{"osd":5,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.66700000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70099999999999996}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.315,"15min":0.28399999999999997},"min":{"1min":0.21299999999999999,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.374,"5min":0.317,"15min":0.28499999999999998},"min":{"1min":0.27300000000000002,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.56200000000000006,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.50700000000000001}]},{"osd":2,"last update":"Sat Mar 30 13:21:52 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.35699999999999998,"15min":0.34699999999999998},"min":{"1min":0.30099999999999999,"5min":0.252,"15min":0.252},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.33100000000000002},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.58599999999999997}]},{"osd":3,"last update":"Sat Mar 30 13:22:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.41399999999999998,"15min":0.41699999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.624},{"interface":"front","average":{"1min":0.438,"5min":0.44500000000000001,"15min":0.44600000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60099999999999998,"5min":0.624,"15min":0.624},"last":0.69999999999999996}]},{"osd":4,"last update":"Sat Mar 30 13:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Mar 30 13:22:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.752},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.78200000000000003}]},{"osd":6,"last update":"Sat Mar 30 13:22:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":13.631,"5min":13.631,"15min":13.631},"last":0.82599999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79800000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 30 13:22:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.35699999999999998,"15min":0.34100000000000003},"min":{"1min":0.28199999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51100000000000001,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.36699999999999999,"15min":0.34499999999999997},"min":{"1min":0.29399999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.57199999999999995,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Mar 30 13:21:54 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39200000000000002},"min":{"1min":0.307,"5min":0.254,"15min":0.254},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.41399999999999998,"15min":0.40799999999999997},"min":{"1min":0.32700000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.627,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.441}]},{"osd":3,"last update":"Sat Mar 30 13:22:18 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.442,"15min":0.443},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.47499999999999998,"15min":0.47299999999999998},"min":{"1min":0.375,"5min":0.37,"15min":0.37},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.46700000000000003}]},{"osd":4,"last update":"Sat Mar 30 13:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.65300000000000002}]},{"osd":5,"last update":"Sat Mar 30 13:22:04 2024","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.4670000000000001,"15min":1.4670000000000001},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":15.381,"5min":15.381,"15min":15.381},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.80100000000000005}]},{"osd":6,"last update":"Sat Mar 30 13:22:29 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.86899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.91100000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:21:50 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.317,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.35599999999999998,"15min":0.34799999999999998},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Mar 30 13:21:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.27500000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.52600000000000002,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.39000000000000001,"15min":0.38200000000000001},"min":{"1min":0.34599999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.35999999999999999}]},{"osd":3,"last update":"Sat Mar 30 13:22:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.435,"15min":0.433},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.38}]},{"osd":4,"last update":"Sat Mar 30 13:21:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.627}]},{"osd":5,"last update":"Sat Mar 30 13:22:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Mar 30 13:22:27 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.73199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75600000000000001}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:19 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40300000000000002,"15min":0.40400000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.376},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.41399999999999998,"15min":0.41599999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Mar 30 13:22:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.441,"15min":0.44400000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.438},{"interface":"front","average":{"1min":0.437,"5min":0.432,"15min":0.43099999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.41999999999999998}]},{"osd":2,"last update":"Sat Mar 30 13:22:19 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.45500000000000002,"15min":0.45600000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.441,"15min":0.443},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.42699999999999999}]},{"osd":4,"last update":"Sat Mar 30 13:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.071,"5min":1.071,"15min":1.071},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.59799999999999998}]},{"osd":5,"last update":"Sat Mar 30 13:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.57399999999999995}]},{"osd":6,"last update":"Sat Mar 30 13:22:29 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.58699999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61499999999999999}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:21:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.73199999999999998}]},{"osd":1,"last update":"Sat Mar 30 13:21:37 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.57899999999999996}]},{"osd":2,"last update":"Sat Mar 30 13:21:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.65000000000000002}]},{"osd":3,"last update":"Sat Mar 30 13:21:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.69399999999999995}]},{"osd":5,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.439}]},{"osd":6,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.52000000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53800000000000003}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.499},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.434}]},{"osd":1,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.58899999999999997}]},{"osd":3,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.67900000000000005}]},{"osd":4,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.62},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.63300000000000001}]},{"osd":6,"last update":"Sat Mar 30 13:22:28 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.57599999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-30T13:22:42.411 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-03-30T13:22:42.630 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:43.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:43 smithi159 ceph-mon[24787]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:22:43.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:43 smithi159 ceph-mon[24787]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:43 smithi187 ceph-mon[27499]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:22:43.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:43 smithi187 ceph-mon[27499]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:44.376 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:22:44.378 INFO:teuthology.orchestra.run.smithi159.stderr:dumped all 2024-03-30T13:22:44.679 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:44 smithi159 ceph-mon[24787]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:44 smithi187 ceph-mon[27499]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:45.028 INFO:teuthology.orchestra.run.smithi159.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-03-30T13:22:44.123320+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48936,"kb_used_data":6376,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912280,"statfs":{"total":767960285184,"available":767910174720,"internally_reserved":0,"allocated":6529024,"data_stored":3571320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-03-30T13:21:51.483087+0000","last_change":"2024-03-30T13:21:51.483087+0000","last_active":"2024-03-30T13:21:51.483087+0000","last_peered":"2024-03-30T13:21:51.483087+0000","last_clean":"2024-03-30T13:21:51.483087+0000","last_became_active":"2024-03-30T13:21:51.482618+0000","last_became_peered":"2024-03-30T13:21:51.482618+0000","last_unstale":"2024-03-30T13:21:51.483087+0000","last_undegraded":"2024-03-30T13:21:51.483087+0000","last_fullsized":"2024-03-30T13:21:51.483087+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-30T13:19:50.443163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-30T13:19:50.443163+0000","last_clean_scrub_stamp":"2024-03-30T13:19:50.443163+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-31T16:56:46.793610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59299999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61499999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42799999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51400000000000001}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":776554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.244,"5min":1.244,"15min":1.244},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":13.776999999999999,"5min":13.776999999999999,"15min":13.776999999999999},"last":0.45600000000000002}]},{"osd":1,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.234,"15min":1.234},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":13.792,"5min":13.792,"15min":13.792},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.57199999999999995}]},{"osd":2,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.52500000000000002}]},{"osd":3,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.495},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.34799999999999998}]},{"osd":5,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.439}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.315,"15min":0.28399999999999997},"min":{"1min":0.21299999999999999,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.374,"5min":0.317,"15min":0.28499999999999998},"min":{"1min":0.27300000000000002,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.56200000000000006,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Mar 30 13:21:52 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.35699999999999998,"15min":0.34699999999999998},"min":{"1min":0.30099999999999999,"5min":0.252,"15min":0.252},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.33100000000000002},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.28999999999999998}]},{"osd":3,"last update":"Sat Mar 30 13:22:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.41399999999999998,"15min":0.41699999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.44500000000000001,"15min":0.44600000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60099999999999998,"5min":0.624,"15min":0.624},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Mar 30 13:22:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83099999999999996,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.56100000000000005,"15min":0.55700000000000005},"min":{"1min":0.38800000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.50600000000000001}]},{"osd":5,"last update":"Sat Mar 30 13:22:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Mar 30 13:22:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":13.631,"5min":13.631,"15min":13.631},"last":0.48999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 30 13:22:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.35699999999999998,"15min":0.34100000000000003},"min":{"1min":0.28199999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51100000000000001,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.36699999999999999,"15min":0.34499999999999997},"min":{"1min":0.29399999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.57199999999999995,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Mar 30 13:21:54 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39200000000000002},"min":{"1min":0.307,"5min":0.254,"15min":0.254},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.41399999999999998,"15min":0.40799999999999997},"min":{"1min":0.32700000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.627,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.441}]},{"osd":3,"last update":"Sat Mar 30 13:22:18 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.442,"15min":0.443},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.47499999999999998,"15min":0.47299999999999998},"min":{"1min":0.375,"5min":0.37,"15min":0.37},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.46700000000000003}]},{"osd":4,"last update":"Sat Mar 30 13:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.65300000000000002}]},{"osd":5,"last update":"Sat Mar 30 13:22:04 2024","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.4670000000000001,"15min":1.4670000000000001},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":15.381,"5min":15.381,"15min":15.381},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.80100000000000005}]},{"osd":6,"last update":"Sat Mar 30 13:22:29 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.86899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.91100000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:21:50 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.317,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.35599999999999998,"15min":0.34799999999999998},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Mar 30 13:21:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.27500000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.52600000000000002,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.39000000000000001,"15min":0.38200000000000001},"min":{"1min":0.34599999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.35999999999999999}]},{"osd":3,"last update":"Sat Mar 30 13:22:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.435,"15min":0.433},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.38}]},{"osd":4,"last update":"Sat Mar 30 13:21:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.627}]},{"osd":5,"last update":"Sat Mar 30 13:22:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Mar 30 13:22:27 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.73199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75600000000000001}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:19 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40300000000000002,"15min":0.40400000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.41399999999999998,"15min":0.41599999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.39500000000000002}]},{"osd":1,"last update":"Sat Mar 30 13:22:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.441,"15min":0.44400000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.437,"5min":0.432,"15min":0.43099999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Mar 30 13:22:19 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.45500000000000002,"15min":0.45600000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.441,"15min":0.443},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.36599999999999999}]},{"osd":4,"last update":"Sat Mar 30 13:22:41 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.60199999999999998,"15min":0.59599999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1,"5min":1.071,"15min":1.071},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.625,"5min":0.59199999999999997,"15min":0.58599999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.71499999999999997}]},{"osd":5,"last update":"Sat Mar 30 13:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Mar 30 13:22:29 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.80400000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61299999999999999}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.38900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.72299999999999998,"5min":0.747,"15min":0.747},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52600000000000002,"15min":0.52900000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.75600000000000001,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.66900000000000004}]},{"osd":1,"last update":"Sat Mar 30 13:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.5,"15min":0.499},"min":{"1min":0.36199999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.434,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64800000000000002,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.59799999999999998}]},{"osd":2,"last update":"Sat Mar 30 13:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.75700000000000001,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.53500000000000003,"15min":0.53200000000000003},"min":{"1min":0.37,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.74099999999999999}]},{"osd":3,"last update":"Sat Mar 30 13:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.70199999999999996,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.56599999999999995,"15min":0.56999999999999995},"min":{"1min":0.40600000000000003,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.69399999999999995,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.86199999999999999}]},{"osd":5,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.82299999999999995}]},{"osd":6,"last update":"Sat Mar 30 13:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.79900000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69599999999999995}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.502},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.58299999999999996}]},{"osd":1,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.442}]},{"osd":2,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.47699999999999998}]},{"osd":4,"last update":"Sat Mar 30 13:22:02 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Mar 30 13:22:28 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.40899999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-30T13:22:45.030 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-30T13:22:45.030 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-30T13:22:45.030 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-30T13:22:45.033 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph health --format=json 2024-03-30T13:22:45.250 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/mon.smithi159/config 2024-03-30T13:22:45.639 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:45 smithi159 ceph-mon[24787]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:22:45.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:45 smithi187 ceph-mon[27499]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:22:46.638 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:46 smithi159 ceph-mon[24787]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:46.638 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:22:46.638 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:22:46.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:46 smithi187 ceph-mon[27499]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:22:46.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:22:47.042 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:22:47.042 INFO:teuthology.orchestra.run.smithi159.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-30T13:22:47.624 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-30T13:22:47.624 INFO:tasks.cephadm:Setup complete, yielding 2024-03-30T13:22:47.624 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-30T13:22:47.634 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:22:47.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch status' 2024-03-30T13:22:47.660 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:47 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3696563662' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-30T13:22:47.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:47 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3696563662' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-30T13:22:48.605 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:48 smithi159 ceph-mon[24787]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:48.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:48 smithi187 ceph-mon[27499]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:49.495 INFO:teuthology.orchestra.run.smithi159.stdout:Backend: cephadm 2024-03-30T13:22:49.495 INFO:teuthology.orchestra.run.smithi159.stdout:Available: Yes 2024-03-30T13:22:49.496 INFO:teuthology.orchestra.run.smithi159.stdout:Paused: No 2024-03-30T13:22:49.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:22:49.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:22:49.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:22:49.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:49 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:22:49.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:22:49.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:22:49.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:22:49.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:49 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:22:50.094 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch ps' 2024-03-30T13:22:50.595 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:50 smithi159 ceph-mon[24787]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:22:50.596 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:50 smithi159 ceph-mon[24787]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:50.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:50 smithi187 ceph-mon[27499]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:22:50.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:50 smithi187 ceph-mon[27499]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:52.046 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (4m) 2m ago 5m 15.4M - ba2b418f427c a3dca4f7bb16 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (5m) 2m ago 5m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (4m) 66s ago 4m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (3m) 2m ago 5m 44.9M - 8.3.5 dad864ee21e9 3e8ed3a7ad2a 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 2fcdf09abf3a 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (4m) 66s ago 4m 411M - 17.2.0 e1d6a67b021e 6d26676209f5 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (6m) 2m ago 6m 42.7M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (4m) 66s ago 4m 39.8M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (5m) 2m ago 5m 17.0M - 1dbe0e931976 c7e53de4dadb 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (4m) 66s ago 4m 19.6M - 1dbe0e931976 71a2445fdf72 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (4m) 2m ago 4m 36.2M 3216M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:22:52.047 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (3m) 2m ago 3m 36.6M 3216M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:22:52.048 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (3m) 2m ago 3m 34.4M 3216M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:22:52.048 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (2m) 2m ago 2m 27.1M 3216M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:22:52.048 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (2m) 66s ago 2m 36.7M 3984M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:22:52.048 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (117s) 66s ago 116s 32.3M 3984M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:22:52.048 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (94s) 66s ago 93s 31.4M 3984M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:22:52.048 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (70s) 66s ago 70s 10.9M 3984M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:22:52.048 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (4m) 2m ago 4m 55.7M - 514e6a882f6e 9863c014034e 2024-03-30T13:22:52.641 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch ls' 2024-03-30T13:22:53.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:53 smithi159 ceph-mon[24787]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:22:53.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:53 smithi159 ceph-mon[24787]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:53.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:53 smithi187 ceph-mon[27499]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:22:53.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:53 smithi187 ceph-mon[27499]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:crash 2/2 2m ago 6m * 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:mgr 2/2 2m ago 6m count:2 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:mon 2/2 2m ago 5m smithi159:172.21.15.159=smithi159;smithi187:172.21.15.187=smithi187;count:2 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:osd 8 2m ago - 2024-03-30T13:22:54.538 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-03-30T13:22:54.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:54 smithi159 ceph-mon[24787]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:54.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:54 smithi187 ceph-mon[27499]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:55.245 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch host ls' 2024-03-30T13:22:55.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:55 smithi159 ceph-mon[24787]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:22:55.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:55 smithi187 ceph-mon[27499]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:22:56.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:56 smithi159 ceph-mon[24787]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:56.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:56 smithi187 ceph-mon[27499]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:57.197 INFO:teuthology.orchestra.run.smithi159.stdout:HOST ADDR LABELS STATUS 2024-03-30T13:22:57.197 INFO:teuthology.orchestra.run.smithi159.stdout:smithi159 172.21.15.159 2024-03-30T13:22:57.197 INFO:teuthology.orchestra.run.smithi159.stdout:smithi187 172.21.15.187 2024-03-30T13:22:57.197 INFO:teuthology.orchestra.run.smithi159.stdout:2 hosts in cluster 2024-03-30T13:22:57.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:57 smithi159 ceph-mon[24787]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:22:57.764 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch device ls' 2024-03-30T13:22:57.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:57 smithi187 ceph-mon[27499]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:22:58.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:22:58 smithi159 ceph-mon[24787]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:22:58 smithi187 ceph-mon[27499]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:22:59.551 INFO:teuthology.orchestra.run.smithi159.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-03-30T13:22:59.551 INFO:teuthology.orchestra.run.smithi159.stdout:smithi159 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300GH400BGN 400G LVM detected, locked 2024-03-30T13:22:59.551 INFO:teuthology.orchestra.run.smithi159.stdout:smithi187 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620500MW400BGN 400G LVM detected, locked 2024-03-30T13:23:00.136 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-30T13:23:00.146 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:23:00.146 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-03-30T13:23:00.175 INFO:teuthology.orchestra.run.smithi159.stderr:+ systemctl stop nfs-server 2024-03-30T13:23:00.182 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi187.front.sepia.ceph.com 2024-03-30T13:23:00.183 DEBUG:teuthology.orchestra.run.smithi187:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-03-30T13:23:00.215 INFO:teuthology.orchestra.run.smithi187.stderr:+ systemctl stop nfs-server 2024-03-30T13:23:00.226 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-30T13:23:00.234 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:23:00.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph fs volume create foofs' 2024-03-30T13:23:01.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:01 smithi159 ceph-mon[24787]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:01.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:01 smithi159 ceph-mon[24787]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:01 smithi187 ceph-mon[27499]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:01 smithi187 ceph-mon[27499]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:03.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:03 smithi159 ceph-mon[24787]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:03.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:03 smithi159 ceph-mon[24787]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:03.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:03 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-03-30T13:23:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:03 smithi187 ceph-mon[27499]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:03 smithi187 ceph-mon[27499]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:03 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-03-30T13:23:04.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-03-30T13:23:04.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:04 smithi159 ceph-mon[24787]: osdmap e61: 8 total, 8 up, 8 in 2024-03-30T13:23:04.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-03-30T13:23:04.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:04 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[24782]: 2024-03-30T13:23:04.197+0000 7f8d26365700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:23:04.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-03-30T13:23:04.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:04 smithi187 ceph-mon[27499]: osdmap e61: 8 total, 8 up, 8 in 2024-03-30T13:23:04.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-03-30T13:23:04.881 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-03-30T13:23:04.890 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-03-30T13:23:04.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch ls -f json 2024-03-30T13:23:05.412 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: pgmap v208: 33 pgs: 7 creating+peering, 25 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:05.412 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-03-30T13:23:05.412 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: osdmap e62: 8 total, 8 up, 8 in 2024-03-30T13:23:05.412 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: osdmap e63: 8 total, 8 up, 8 in 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: fsmap foofs:0 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: Saving service mds.foofs spec with placement count:2 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi159.jwwwqm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi159.jwwwqm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-30T13:23:05.413 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:05.414 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:05 smithi159 ceph-mon[24787]: Deploying daemon mds.foofs.smithi159.jwwwqm on smithi159 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: pgmap v208: 33 pgs: 7 creating+peering, 25 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: osdmap e62: 8 total, 8 up, 8 in 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: osdmap e63: 8 total, 8 up, 8 in 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-03-30T13:23:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: fsmap foofs:0 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: Saving service mds.foofs spec with placement count:2 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi159.jwwwqm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi159.jwwwqm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:05.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:05 smithi187 ceph-mon[27499]: Deploying daemon mds.foofs.smithi159.jwwwqm on smithi159 2024-03-30T13:23:06.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:06 smithi159 ceph-mon[24787]: osdmap e64: 8 total, 8 up, 8 in 2024-03-30T13:23:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:06 smithi187 ceph-mon[27499]: osdmap e64: 8 total, 8 up, 8 in 2024-03-30T13:23:07.394 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:07 smithi159 ceph-mon[24787]: pgmap v212: 65 pgs: 19 creating+peering, 38 unknown, 8 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:07.394 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:07 smithi159 ceph-mon[24787]: osdmap e65: 8 total, 8 up, 8 in 2024-03-30T13:23:07.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:07 smithi187 ceph-mon[27499]: pgmap v212: 65 pgs: 19 creating+peering, 38 unknown, 8 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:07.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:07 smithi187 ceph-mon[27499]: osdmap e65: 8 total, 8 up, 8 in 2024-03-30T13:23:08.098 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:23:08.099 INFO:teuthology.orchestra.run.smithi159.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-30T13:16:39.794254Z", "last_refresh": "2024-03-30T13:20:25.166370Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-30T13:16:34.339500Z", "last_refresh": "2024-03-30T13:20:25.166435Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-30T13:16:37.087366Z", "last_refresh": "2024-03-30T13:20:25.166514Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-30T13:23:04.219328Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-30T13:23:04.210243Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-30T13:16:33.126309Z", "last_refresh": "2024-03-30T13:20:25.166297Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi159:172.21.15.159=smithi159", "smithi187:172.21.15.187=smithi187"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-30T13:17:21.867431Z", "last_refresh": "2024-03-30T13:20:25.166153Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-30T13:16:38.180123Z", "last_refresh": "2024-03-30T13:20:25.166573Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-30T13:20:25.166694Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-30T13:16:35.797099Z", "last_refresh": "2024-03-30T13:20:25.166634Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-30T13:23:08.994 INFO:tasks.cephadm:mds.foofs has 0/2 2024-03-30T13:23:09.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: pgmap v214: 65 pgs: 19 creating+peering, 10 unknown, 36 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: daemon mds.foofs.smithi159.jwwwqm assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: Cluster is now healthy 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: mds.? [v2:172.21.15.159:6834/3724310836,v1:172.21.15.159:6835/3724310836] up:boot 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:creating} 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: daemon mds.foofs.smithi159.jwwwqm is now active in filesystem foofs as rank 0 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:09.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi187.bkvdqd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:23:09.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi187.bkvdqd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-30T13:23:09.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:09 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: pgmap v214: 65 pgs: 19 creating+peering, 10 unknown, 36 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: daemon mds.foofs.smithi159.jwwwqm assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: Cluster is now healthy 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: mds.? [v2:172.21.15.159:6834/3724310836,v1:172.21.15.159:6835/3724310836] up:boot 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:creating} 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: daemon mds.foofs.smithi159.jwwwqm is now active in filesystem foofs as rank 0 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:09.326 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi187.bkvdqd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:23:09.327 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi187.bkvdqd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-30T13:23:09.327 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:09 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:09.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch ls -f json 2024-03-30T13:23:10.304 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:10 smithi159 ceph-mon[24787]: Deploying daemon mds.foofs.smithi187.bkvdqd on smithi187 2024-03-30T13:23:10.304 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:10 smithi159 ceph-mon[24787]: mds.? [v2:172.21.15.159:6834/3724310836,v1:172.21.15.159:6835/3724310836] up:active 2024-03-30T13:23:10.305 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:10 smithi159 ceph-mon[24787]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 2024-03-30T13:23:10.471 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:10 smithi187 ceph-mon[27499]: Deploying daemon mds.foofs.smithi187.bkvdqd on smithi187 2024-03-30T13:23:10.471 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:10 smithi187 ceph-mon[27499]: mds.? [v2:172.21.15.159:6834/3724310836,v1:172.21.15.159:6835/3724310836] up:active 2024-03-30T13:23:10.471 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:10 smithi187 ceph-mon[27499]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 2024-03-30T13:23:11.482 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:11 smithi159 ceph-mon[24787]: pgmap v215: 65 pgs: 2 creating+peering, 63 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-03-30T13:23:11.482 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:11 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:11.482 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:11 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:11.483 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:11 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:11.483 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:11 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:11.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:11 smithi187 ceph-mon[27499]: pgmap v215: 65 pgs: 2 creating+peering, 63 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-03-30T13:23:11.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:11 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:11.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:11 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:11.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:11 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:11.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:11 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:12.020 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:23:12.020 INFO:teuthology.orchestra.run.smithi159.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-30T13:16:39.794254Z", "last_refresh": "2024-03-30T13:20:25.166370Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-30T13:16:34.339500Z", "last_refresh": "2024-03-30T13:20:25.166435Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-30T13:16:37.087366Z", "last_refresh": "2024-03-30T13:20:25.166514Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-30T13:23:04.219328Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-30T13:23:04.210243Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-30T13:16:33.126309Z", "last_refresh": "2024-03-30T13:20:25.166297Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi159:172.21.15.159=smithi159", "smithi187:172.21.15.187=smithi187"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-30T13:17:21.867431Z", "last_refresh": "2024-03-30T13:20:25.166153Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-30T13:16:38.180123Z", "last_refresh": "2024-03-30T13:20:25.166573Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-30T13:20:25.166694Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-30T13:16:35.797099Z", "last_refresh": "2024-03-30T13:20:25.166634Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-30T13:23:12.661 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:12 smithi159 ceph-mon[24787]: mds.? [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] up:boot 2024-03-30T13:23:12.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:12 smithi159 ceph-mon[24787]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:23:12.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:12 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:23:12.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:12 smithi159 ceph-mon[24787]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:23:12.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:12 smithi159 ceph-mon[24787]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:23:12.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:12 smithi159 ceph-mon[24787]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-30T13:23:12.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:12 smithi187 ceph-mon[27499]: mds.? [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] up:boot 2024-03-30T13:23:12.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:12 smithi187 ceph-mon[27499]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:23:12.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:12 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:23:12.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:12 smithi187 ceph-mon[27499]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:23:12.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:12 smithi187 ceph-mon[27499]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:23:12.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:12 smithi187 ceph-mon[27499]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-30T13:23:13.938 INFO:tasks.cephadm:mds.foofs has 0/2 2024-03-30T13:23:14.192 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:14 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:14.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:14 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:14.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch ls -f json 2024-03-30T13:23:15.481 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:15 smithi159 ceph-mon[24787]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-30T13:23:15.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:15 smithi187 ceph-mon[27499]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-30T13:23:17.123 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:16 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:17.123 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:16 smithi159 ceph-mon[24787]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-03-30T13:23:17.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:16 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:17.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:16 smithi187 ceph-mon[27499]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-03-30T13:23:17.773 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:23:17.773 INFO:teuthology.orchestra.run.smithi159.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-30T13:16:39.794254Z", "last_refresh": "2024-03-30T13:23:17.704594Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-30T13:16:34.339500Z", "last_refresh": "2024-03-30T13:23:13.021052Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-30T13:16:37.087366Z", "last_refresh": "2024-03-30T13:23:17.704721Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-30T13:23:04.219328Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-30T13:23:04.210243Z", "last_refresh": "2024-03-30T13:23:13.021654Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-30T13:16:33.126309Z", "last_refresh": "2024-03-30T13:23:13.021218Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi159:172.21.15.159=smithi159", "smithi187:172.21.15.187=smithi187"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-30T13:17:21.867431Z", "last_refresh": "2024-03-30T13:23:13.021290Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-30T13:16:38.180123Z", "last_refresh": "2024-03-30T13:23:13.021354Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-30T13:23:13.021415Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-30T13:16:35.797099Z", "last_refresh": "2024-03-30T13:23:17.704844Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-30T13:23:18.439 INFO:tasks.cephadm:mds.foofs has 2/2 2024-03-30T13:23:18.439 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-30T13:23:18.453 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:23:18.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-03-30T13:23:18.983 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:18.983 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:18.983 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:18.983 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:18.983 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:18.983 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:18 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:18.983 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:18 smithi159 ceph-mon[24787]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:23:18.983 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:18 smithi159 ceph-mon[24787]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-30T13:23:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:18 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:18 smithi187 ceph-mon[27499]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:23:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:18 smithi187 ceph-mon[27499]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-30T13:23:21.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:21 smithi159 ceph-mon[24787]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-30T13:23:21.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:21 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-03-30T13:23:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:21 smithi187 ceph-mon[27499]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-30T13:23:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:21 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-03-30T13:23:22.230 INFO:teuthology.orchestra.run.smithi159.stdout:NFS Cluster Created Successfully 2024-03-30T13:23:22.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:22 smithi159 ceph-mon[24787]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:22.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-03-30T13:23:22.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:22 smithi159 ceph-mon[24787]: osdmap e66: 8 total, 8 up, 8 in 2024-03-30T13:23:22.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:22 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-03-30T13:23:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:22 smithi187 ceph-mon[27499]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-03-30T13:23:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:22 smithi187 ceph-mon[27499]: osdmap e66: 8 total, 8 up, 8 in 2024-03-30T13:23:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:22 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-03-30T13:23:22.918 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-03-30T13:23:23.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: pgmap v222: 97 pgs: 5 creating+peering, 27 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-03-30T13:23:23.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-03-30T13:23:23.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: osdmap e67: 8 total, 8 up, 8 in 2024-03-30T13:23:23.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: Saving service nfs.foo spec with placement count:2 2024-03-30T13:23:23.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:23.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:23.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:23.422 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: Creating key for client.nfs.foo.0.0.smithi159.vhnwtw 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-30T13:23:23.423 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:23 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: pgmap v222: 97 pgs: 5 creating+peering, 27 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: osdmap e67: 8 total, 8 up, 8 in 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: Saving service nfs.foo spec with placement count:2 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: Creating key for client.nfs.foo.0.0.smithi159.vhnwtw 2024-03-30T13:23:23.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-30T13:23:23.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:23 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:24.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:24 smithi159 ceph-mon[24787]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-30T13:23:24.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:24 smithi159 ceph-mon[24787]: Rados config object exists: conf-nfs.foo 2024-03-30T13:23:24.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:24 smithi159 ceph-mon[24787]: Creating key for client.nfs.foo.0.0.smithi159.vhnwtw-rgw 2024-03-30T13:23:24.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:24 smithi159 ceph-mon[24787]: Deploying daemon nfs.foo.0.0.smithi159.vhnwtw on smithi159 2024-03-30T13:23:24.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:24 smithi159 ceph-mon[24787]: osdmap e68: 8 total, 8 up, 8 in 2024-03-30T13:23:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:24 smithi187 ceph-mon[27499]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-30T13:23:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:24 smithi187 ceph-mon[27499]: Rados config object exists: conf-nfs.foo 2024-03-30T13:23:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:24 smithi187 ceph-mon[27499]: Creating key for client.nfs.foo.0.0.smithi159.vhnwtw-rgw 2024-03-30T13:23:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:24 smithi187 ceph-mon[27499]: Deploying daemon nfs.foo.0.0.smithi159.vhnwtw on smithi159 2024-03-30T13:23:24.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:24 smithi187 ceph-mon[27499]: osdmap e68: 8 total, 8 up, 8 in 2024-03-30T13:23:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:25 smithi159 ceph-mon[24787]: pgmap v225: 97 pgs: 6 creating+activating, 5 creating+peering, 9 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-30T13:23:25.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:25 smithi159 ceph-mon[24787]: mgrmap e20: smithi159.fnasly(active, since 5m), standbys: smithi187.jpkaud 2024-03-30T13:23:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:25 smithi187 ceph-mon[27499]: pgmap v225: 97 pgs: 6 creating+activating, 5 creating+peering, 9 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-30T13:23:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:25 smithi187 ceph-mon[27499]: mgrmap e20: smithi159.fnasly(active, since 5m), standbys: smithi187.jpkaud 2024-03-30T13:23:26.270 INFO:teuthology.orchestra.run.smithi159.stderr:Invalid command: Unexpected argument '--clusterid' 2024-03-30T13:23:26.270 INFO:teuthology.orchestra.run.smithi159.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-03-30T13:23:26.270 INFO:teuthology.orchestra.run.smithi159.stderr:Error EINVAL: invalid command 2024-03-30T13:23:26.629 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:23:26.629 INFO:teuthology.orchestra.run.smithi159.stdout: "bind": "/fake", 2024-03-30T13:23:26.629 INFO:teuthology.orchestra.run.smithi159.stdout: "fs": "foofs", 2024-03-30T13:23:26.629 INFO:teuthology.orchestra.run.smithi159.stdout: "path": "/", 2024-03-30T13:23:26.629 INFO:teuthology.orchestra.run.smithi159.stdout: "cluster": "foo", 2024-03-30T13:23:26.629 INFO:teuthology.orchestra.run.smithi159.stdout: "mode": "RW" 2024-03-30T13:23:26.630 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:23:27.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-03-30T13:23:27.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: pgmap v226: 97 pgs: 6 creating+activating, 5 creating+peering, 86 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-30T13:23:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-03-30T13:23:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-03-30T13:23:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-30T13:23:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-30T13:23:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-30T13:23:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-30T13:23:27.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:27 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: pgmap v226: 97 pgs: 6 creating+activating, 5 creating+peering, 86 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-30T13:23:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-03-30T13:23:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-03-30T13:23:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:27.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-30T13:23:27.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-30T13:23:27.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-30T13:23:27.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-30T13:23:27.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:27 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:28.474 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:28 smithi159 ceph-mon[24787]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:28.475 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:28 smithi159 ceph-mon[24787]: Creating key for client.nfs.foo.1.0.smithi187.dwmvfp 2024-03-30T13:23:28.475 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:28 smithi159 ceph-mon[24787]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-30T13:23:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:28 smithi187 ceph-mon[27499]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:28 smithi187 ceph-mon[27499]: Creating key for client.nfs.foo.1.0.smithi187.dwmvfp 2024-03-30T13:23:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:28 smithi187 ceph-mon[27499]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-30T13:23:29.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:29 smithi159 ceph-mon[24787]: pgmap v227: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-03-30T13:23:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:29 smithi187 ceph-mon[27499]: pgmap v227: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-03-30T13:23:30.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:30 smithi187 ceph-mon[27499]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:30.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:30 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-30T13:23:30.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:30 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-30T13:23:30.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:30 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-30T13:23:30.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:30 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-30T13:23:30.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:30 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:30.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:30 smithi159 ceph-mon[24787]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:30.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:30 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-30T13:23:30.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:30 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-30T13:23:30.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:30 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-30T13:23:30.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:30 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-30T13:23:30.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:30 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:31.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:31 smithi187 ceph-mon[27499]: Rados config object exists: conf-nfs.foo 2024-03-30T13:23:31.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:31 smithi187 ceph-mon[27499]: Creating key for client.nfs.foo.1.0.smithi187.dwmvfp-rgw 2024-03-30T13:23:31.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:31 smithi187 ceph-mon[27499]: Deploying daemon nfs.foo.1.0.smithi187.dwmvfp on smithi187 2024-03-30T13:23:31.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:31 smithi187 ceph-mon[27499]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-30T13:23:31.481 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:31 smithi159 ceph-mon[24787]: Rados config object exists: conf-nfs.foo 2024-03-30T13:23:31.481 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:31 smithi159 ceph-mon[24787]: Creating key for client.nfs.foo.1.0.smithi187.dwmvfp-rgw 2024-03-30T13:23:31.481 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:31 smithi159 ceph-mon[24787]: Deploying daemon nfs.foo.1.0.smithi187.dwmvfp on smithi187 2024-03-30T13:23:31.481 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:31 smithi159 ceph-mon[24787]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-30T13:23:32.498 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:32 smithi159 ceph-mon[24787]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:32.498 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:32 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:32.498 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:32 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:32.498 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:32 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:32.499 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:32 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:32.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:32 smithi187 ceph-mon[27499]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:32.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:32 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:32.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:32 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:32.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:32 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:32.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:32 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:33.462 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:33 smithi187 ceph-mon[27499]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:33.463 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:33 smithi187 ceph-mon[27499]: pgmap v229: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-03-30T13:23:33.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:33 smithi159 ceph-mon[24787]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:33.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:33 smithi159 ceph-mon[24787]: pgmap v229: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-03-30T13:23:34.238 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:34 smithi159 ceph-mon[24787]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:34.506 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:34 smithi187 ceph-mon[27499]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:35.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:35 smithi159 ceph-mon[24787]: pgmap v230: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-03-30T13:23:35.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:35 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:35.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:35 smithi187 ceph-mon[27499]: pgmap v230: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-03-30T13:23:35.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:35 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:36.495 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:36 smithi159 ceph-mon[24787]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:36.495 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:36 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:36.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:36 smithi187 ceph-mon[27499]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:36.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:36 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:36.995 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-03-30T13:23:37.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:37 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-03-30T13:23:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:23:37.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:23:37.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:23:37.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:37 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:23:37.757 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-30T13:23:37.768 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:23:37.769 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-03-30T13:23:37.800 INFO:teuthology.orchestra.run.smithi159.stderr:+ mkdir /mnt/foo 2024-03-30T13:23:37.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-03-30T13:23:37.880 INFO:teuthology.orchestra.run.smithi159.stderr:++ hostname 2024-03-30T13:23:37.881 INFO:teuthology.orchestra.run.smithi159.stderr:+ mount -t nfs smithi159:/fake /mnt/foo -o sync 2024-03-30T13:23:38.268 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:38 smithi159 ceph-mon[24787]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:38.286 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-03-30T13:23:38.353 INFO:teuthology.orchestra.run.smithi159.stderr:+ echo test 2024-03-30T13:23:38.379 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-03-30T13:23:38.446 INFO:teuthology.orchestra.run.smithi159.stderr:+ sync 2024-03-30T13:23:38.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:38 smithi187 ceph-mon[27499]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:23:38.671 INFO:teuthology.run_tasks:Running task parallel... 2024-03-30T13:23:38.680 INFO:teuthology.task.parallel:starting parallel... 2024-03-30T13:23:38.681 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-30T13:23:38.681 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-30T13:23:38.681 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:23:38.682 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-30T13:23:38.682 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-30T13:23:38.682 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-03-30T13:23:38.683 INFO:teuthology.task.exec:Executing custom commands... 2024-03-30T13:23:38.683 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:23:38.683 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-03-30T13:23:38.760 INFO:teuthology.orchestra.run.smithi159.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-30T13:23:38.760 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:23:38.760 INFO:teuthology.orchestra.run.smithi159.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-03-30T13:23:38.761 INFO:teuthology.orchestra.run.smithi159.stdout:3 of 5 processes prepared for launch 0 sec 2024-03-30T13:23:38.862 INFO:teuthology.orchestra.run.smithi159.stdout:5 of 5 processes prepared for launch 0 sec 2024-03-30T13:23:38.862 INFO:teuthology.orchestra.run.smithi159.stdout:releasing clients 2024-03-30T13:23:39.449 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:39 smithi159 ceph-mon[24787]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-03-30T13:23:39.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:39 smithi187 ceph-mon[27499]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-03-30T13:23:39.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 130 28.75 MB/sec warmup 1 sec latency 38.902 ms 2024-03-30T13:23:40.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 257 24.29 MB/sec warmup 2 sec latency 38.628 ms 2024-03-30T13:23:41.280 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:41 smithi159 ceph-mon[24787]: pgmap v233: 97 pgs: 97 active+clean; 2.6 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 173 KiB/s wr, 7 op/s 2024-03-30T13:23:41.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:41 smithi187 ceph-mon[27499]: pgmap v233: 97 pgs: 97 active+clean; 2.6 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 173 KiB/s wr, 7 op/s 2024-03-30T13:23:41.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 392 23.19 MB/sec warmup 3 sec latency 31.425 ms 2024-03-30T13:23:42.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-30T13:23:42.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 559 23.91 MB/sec warmup 4 sec latency 30.096 ms 2024-03-30T13:23:43.428 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:43 smithi159 ceph-mon[24787]: pgmap v234: 97 pgs: 97 active+clean; 16 MiB data, 61 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.4 MiB/s wr, 85 op/s 2024-03-30T13:23:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:43 smithi187 ceph-mon[27499]: pgmap v234: 97 pgs: 97 active+clean; 16 MiB data, 61 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.4 MiB/s wr, 85 op/s 2024-03-30T13:23:43.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 888 21.66 MB/sec warmup 5 sec latency 30.312 ms 2024-03-30T13:23:44.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:44 smithi159 ceph-mon[24787]: pgmap v235: 97 pgs: 97 active+clean; 78 MiB data, 127 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 6.6 MiB/s wr, 221 op/s 2024-03-30T13:23:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:44 smithi187 ceph-mon[27499]: pgmap v235: 97 pgs: 97 active+clean; 78 MiB data, 127 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 6.6 MiB/s wr, 221 op/s 2024-03-30T13:23:44.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 1418 21.68 MB/sec warmup 6 sec latency 31.033 ms 2024-03-30T13:23:45.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 2375 19.96 MB/sec warmup 7 sec latency 20.195 ms 2024-03-30T13:23:46.228 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-03-30T13:23:46.228 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-03-30T13:23:46.228 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-30T13:23:46.228 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-03-30T13:23:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-03-30T13:23:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-03-30T13:23:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-30T13:23:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-03-30T13:23:46.621 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-30T13:23:46.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 3280 21.96 MB/sec warmup 8 sec latency 21.398 ms 2024-03-30T13:23:47.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: pgmap v236: 97 pgs: 97 active+clean; 99 MiB data, 168 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 8.6 MiB/s wr, 326 op/s 2024-03-30T13:23:47.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-03-30T13:23:47.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-03-30T13:23:47.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-03-30T13:23:47.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-03-30T13:23:47.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: osdmap e69: 8 total, 8 up, 8 in 2024-03-30T13:23:47.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:23:47.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:23:47.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:47 smithi159 ceph-mon[24787]: osdmap e70: 8 total, 8 up, 8 in 2024-03-30T13:23:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: pgmap v236: 97 pgs: 97 active+clean; 99 MiB data, 168 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 8.6 MiB/s wr, 326 op/s 2024-03-30T13:23:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-03-30T13:23:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-03-30T13:23:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-03-30T13:23:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-03-30T13:23:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: osdmap e69: 8 total, 8 up, 8 in 2024-03-30T13:23:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:23:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:23:47.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:47 smithi187 ceph-mon[27499]: osdmap e70: 8 total, 8 up, 8 in 2024-03-30T13:23:47.862 INFO:teuthology.orchestra.run.smithi159.stdout: 5 3918 22.22 MB/sec warmup 9 sec latency 20.728 ms 2024-03-30T13:23:48.863 INFO:teuthology.orchestra.run.smithi159.stdout: 5 4753 21.07 MB/sec warmup 10 sec latency 14.657 ms 2024-03-30T13:23:49.249 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:49 smithi159 ceph-mon[24787]: pgmap v239: 97 pgs: 97 active+clean; 115 MiB data, 195 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 15 MiB/s wr, 700 op/s 2024-03-30T13:23:49.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:49 smithi187 ceph-mon[27499]: pgmap v239: 97 pgs: 97 active+clean; 115 MiB data, 195 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 15 MiB/s wr, 700 op/s 2024-03-30T13:23:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: 5 5768 20.94 MB/sec warmup 11 sec latency 57.994 ms 2024-03-30T13:23:50.863 INFO:teuthology.orchestra.run.smithi159.stdout: 5 6533 21.87 MB/sec warmup 12 sec latency 31.326 ms 2024-03-30T13:23:51.260 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:51 smithi159 ceph-mon[24787]: pgmap v240: 97 pgs: 97 active+clean; 132 MiB data, 272 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 18 MiB/s wr, 972 op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:51.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:51 smithi187 ceph-mon[27499]: pgmap v240: 97 pgs: 97 active+clean; 132 MiB data, 272 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 18 MiB/s wr, 972 op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:51.863 INFO:teuthology.orchestra.run.smithi159.stdout: 5 7479 22.75 MB/sec warmup 13 sec latency 13.277 ms 2024-03-30T13:23:52.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph mgr module enable nfs --force' 2024-03-30T13:23:52.863 INFO:teuthology.orchestra.run.smithi159.stdout: 5 8176 21.70 MB/sec warmup 14 sec latency 21.633 ms 2024-03-30T13:23:53.276 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:53 smithi159 ceph-mon[24787]: pgmap v241: 97 pgs: 97 active+clean; 137 MiB data, 275 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 11 MiB/s wr, 931 op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:53.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:53 smithi187 ceph-mon[27499]: pgmap v241: 97 pgs: 97 active+clean; 137 MiB data, 275 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 11 MiB/s wr, 931 op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:53.863 INFO:teuthology.orchestra.run.smithi159.stdout: 5 9183 21.68 MB/sec warmup 15 sec latency 18.420 ms 2024-03-30T13:23:54.537 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:54 smithi159 ceph-mon[24787]: pgmap v242: 97 pgs: 97 active+clean; 151 MiB data, 328 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 14 MiB/s wr, 1.31k op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:54.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:54 smithi187 ceph-mon[27499]: pgmap v242: 97 pgs: 97 active+clean; 151 MiB data, 328 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 14 MiB/s wr, 1.31k op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:54.863 INFO:teuthology.orchestra.run.smithi159.stdout: 5 10118 22.45 MB/sec warmup 16 sec latency 21.270 ms 2024-03-30T13:23:55.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 10790 22.72 MB/sec warmup 17 sec latency 23.422 ms 2024-03-30T13:23:56.118 INFO:teuthology.orchestra.run.smithi159.stderr:module 'nfs' is already enabled 2024-03-30T13:23:56.322 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:56 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1141257719' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-30T13:23:56.322 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:56 smithi159 ceph-mon[24787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-30T13:23:56.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:56 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1141257719' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-30T13:23:56.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:56 smithi187 ceph-mon[27499]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-30T13:23:56.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 11410 21.90 MB/sec warmup 18 sec latency 25.871 ms 2024-03-30T13:23:57.258 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:57 smithi159 ceph-mon[24787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-03-30T13:23:57.258 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:57 smithi159 ceph-mon[24787]: mgrmap e21: smithi159.fnasly(active, since 6m), standbys: smithi187.jpkaud 2024-03-30T13:23:57.259 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:57 smithi159 ceph-mon[24787]: pgmap v243: 97 pgs: 97 active+clean; 149 MiB data, 382 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.10k op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:57.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-30T13:23:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:57 smithi187 ceph-mon[27499]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-03-30T13:23:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:57 smithi187 ceph-mon[27499]: mgrmap e21: smithi159.fnasly(active, since 6m), standbys: smithi187.jpkaud 2024-03-30T13:23:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:57 smithi187 ceph-mon[27499]: pgmap v243: 97 pgs: 97 active+clean; 149 MiB data, 382 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.10k op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:57.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 12323 21.99 MB/sec warmup 19 sec latency 22.765 ms 2024-03-30T13:23:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 13312 21.88 MB/sec warmup 20 sec latency 14.523 ms 2024-03-30T13:23:59.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:23:59 smithi159 ceph-mon[24787]: pgmap v244: 97 pgs: 97 active+clean; 147 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 9.5 KiB/s rd, 12 MiB/s wr, 1.11k op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:59.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:23:59 smithi187 ceph-mon[27499]: pgmap v244: 97 pgs: 97 active+clean; 147 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 9.5 KiB/s rd, 12 MiB/s wr, 1.11k op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:23:59.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 14254 22.65 MB/sec warmup 21 sec latency 16.797 ms 2024-03-30T13:24:00.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 14854 22.20 MB/sec warmup 22 sec latency 51.797 ms 2024-03-30T13:24:01.412 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:01 smithi159 ceph-mon[24787]: pgmap v245: 97 pgs: 97 active+clean; 156 MiB data, 443 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.15k op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:24:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:01 smithi187 ceph-mon[27499]: pgmap v245: 97 pgs: 97 active+clean; 156 MiB data, 443 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.15k op/s; 0 B/s, 0 objects/s recovering 2024-03-30T13:24:01.850 INFO:teuthology.orchestra.run.smithi159.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb 2024-03-30T13:24:01.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 15495 21.89 MB/sec warmup 23 sec latency 84.640 ms 2024-03-30T13:24:02.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 16535 21.81 MB/sec warmup 24 sec latency 12.230 ms 2024-03-30T13:24:02.918 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:02.918 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb 2024-03-30T13:24:02.918 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:02.918 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:24:02.918 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:24:02.918 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:24:02.918 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: pgmap v246: 97 pgs: 97 active+clean; 156 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 924 op/s 2024-03-30T13:24:02.919 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:02.919 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:02 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb 2024-03-30T13:24:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:24:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:24:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:24:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: pgmap v246: 97 pgs: 97 active+clean; 156 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 924 op/s 2024-03-30T13:24:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:02 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:03.132 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-30T13:24:03.133 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:24:03.133 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-03-30T13:24:03.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 17508 22.51 MB/sec warmup 25 sec latency 17.062 ms 2024-03-30T13:24:03.923 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:03 smithi159 ceph-mon[24787]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb 2024-03-30T13:24:04.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:03 smithi187 ceph-mon[27499]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb 2024-03-30T13:24:04.864 INFO:teuthology.orchestra.run.smithi159.stdout: 5 18088 22.33 MB/sec warmup 26 sec latency 26.413 ms 2024-03-30T13:24:04.937 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:04 smithi159 ceph-mon[24787]: pgmap v247: 97 pgs: 97 active+clean; 171 MiB data, 535 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.14k op/s 2024-03-30T13:24:05.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:04 smithi187 ceph-mon[27499]: pgmap v247: 97 pgs: 97 active+clean; 171 MiB data, 535 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.14k op/s 2024-03-30T13:24:05.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 18497 21.70 MB/sec warmup 27 sec latency 32.069 ms 2024-03-30T13:24:06.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 19022 21.33 MB/sec warmup 28 sec latency 28.545 ms 2024-03-30T13:24:06.998 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:24:07.205 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:07 smithi159 ceph-mon[24787]: pgmap v248: 97 pgs: 97 active+clean; 175 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 10 MiB/s wr, 883 op/s 2024-03-30T13:24:07.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:07 smithi187 ceph-mon[27499]: pgmap v248: 97 pgs: 97 active+clean; 175 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 10 MiB/s wr, 883 op/s 2024-03-30T13:24:07.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 19642 21.07 MB/sec warmup 29 sec latency 30.630 ms 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (5m) 31s ago 6m 23.0M - ba2b418f427c a3dca4f7bb16 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (6m) 31s ago 6m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (6m) 33s ago 6m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (5m) 31s ago 6m 52.9M - 8.3.5 dad864ee21e9 3e8ed3a7ad2a 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (60s) 31s ago 59s 14.9M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (57s) 33s ago 56s 14.3M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:9283 running (8m) 31s ago 8m 459M - 17.2.0 e1d6a67b021e 2fcdf09abf3a 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (6m) 33s ago 6m 412M - 17.2.0 e1d6a67b021e 6d26676209f5 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (8m) 31s ago 8m 44.7M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:24:07.972 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (6m) 33s ago 6m 38.0M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (41s) 31s ago 41s 49.0M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (36s) 33s ago 36s 50.8M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (6m) 31s ago 6m 19.0M - 1dbe0e931976 c7e53de4dadb 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (5m) 33s ago 5m 21.5M - 1dbe0e931976 71a2445fdf72 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (5m) 31s ago 5m 45.9M 3216M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (4m) 31s ago 4m 44.0M 3216M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (4m) 31s ago 4m 42.4M 3216M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (3m) 31s ago 3m 40.5M 3216M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (3m) 33s ago 3m 43.0M 3984M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (3m) 33s ago 3m 38.7M 3984M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (2m) 33s ago 2m 40.4M 3984M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (2m) 33s ago 2m 40.7M 3984M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:24:07.973 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (5m) 31s ago 5m 62.4M - 514e6a882f6e 9863c014034e 2024-03-30T13:24:08.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:08 smithi159 ceph-mon[24787]: from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:08.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:08 smithi187 ceph-mon[27499]: from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:24:08.536 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:24:08.537 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:24:08.537 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:24:08.537 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:24:08.537 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-30T13:24:08.537 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:24:08.537 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:24:08.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 20209 20.69 MB/sec warmup 30 sec latency 35.190 ms 2024-03-30T13:24:09.050 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:24:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb", 2024-03-30T13:24:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:24:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [], 2024-03-30T13:24:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "", 2024-03-30T13:24:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb image" 2024-03-30T13:24:09.051 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:24:09.216 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:09 smithi159 ceph-mon[24787]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:09.216 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:09 smithi159 ceph-mon[24787]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:09.216 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:09 smithi159 ceph-mon[24787]: pgmap v249: 97 pgs: 97 active+clean; 178 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-03-30T13:24:09.216 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:09 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1422133255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:24:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:09 smithi187 ceph-mon[27499]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:09 smithi187 ceph-mon[27499]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:09 smithi187 ceph-mon[27499]: pgmap v249: 97 pgs: 97 active+clean; 178 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-03-30T13:24:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:09 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1422133255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:24:09.623 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:24:09.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 20703 20.63 MB/sec warmup 31 sec latency 23.764 ms 2024-03-30T13:24:10.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:10 smithi159 ceph-mon[24787]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:10.235 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:10 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/208918027' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:24:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:10 smithi187 ceph-mon[27499]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:10 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/208918027' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:24:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 21555 21.10 MB/sec warmup 32 sec latency 19.938 ms 2024-03-30T13:24:11.377 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:11 smithi159 ceph-mon[24787]: pgmap v250: 97 pgs: 97 active+clean; 176 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-03-30T13:24:11.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:11 smithi187 ceph-mon[27499]: pgmap v250: 97 pgs: 97 active+clean; 176 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-03-30T13:24:11.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 22023 20.63 MB/sec warmup 33 sec latency 44.293 ms 2024-03-30T13:24:12.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 22536 20.36 MB/sec warmup 34 sec latency 28.825 ms 2024-03-30T13:24:13.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:13 smithi159 ceph-mon[24787]: pgmap v251: 97 pgs: 97 active+clean; 179 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 806 op/s 2024-03-30T13:24:13.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:13 smithi187 ceph-mon[27499]: pgmap v251: 97 pgs: 97 active+clean; 179 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 806 op/s 2024-03-30T13:24:13.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 23200 20.21 MB/sec warmup 35 sec latency 22.360 ms 2024-03-30T13:24:14.664 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:14 smithi159 ceph-mon[24787]: pgmap v252: 97 pgs: 97 active+clean; 187 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-03-30T13:24:14.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:14 smithi187 ceph-mon[27499]: pgmap v252: 97 pgs: 97 active+clean; 187 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-03-30T13:24:14.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 23900 20.07 MB/sec warmup 36 sec latency 60.300 ms 2024-03-30T13:24:15.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 24606 20.32 MB/sec warmup 37 sec latency 39.061 ms 2024-03-30T13:24:16.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 25049 20.21 MB/sec warmup 38 sec latency 56.992 ms 2024-03-30T13:24:17.441 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:17 smithi159 ceph-mon[24787]: pgmap v253: 97 pgs: 97 active+clean; 190 MiB data, 705 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-03-30T13:24:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:17 smithi187 ceph-mon[27499]: pgmap v253: 97 pgs: 97 active+clean; 190 MiB data, 705 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-03-30T13:24:17.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 25453 19.88 MB/sec warmup 39 sec latency 35.501 ms 2024-03-30T13:24:18.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 25990 19.68 MB/sec warmup 40 sec latency 31.172 ms 2024-03-30T13:24:19.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:19 smithi159 ceph-mon[24787]: pgmap v254: 97 pgs: 97 active+clean; 188 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 762 op/s 2024-03-30T13:24:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:19 smithi187 ceph-mon[27499]: pgmap v254: 97 pgs: 97 active+clean; 188 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 762 op/s 2024-03-30T13:24:19.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 26651 19.47 MB/sec warmup 41 sec latency 27.188 ms 2024-03-30T13:24:20.865 INFO:teuthology.orchestra.run.smithi159.stdout: 5 27420 19.45 MB/sec warmup 42 sec latency 23.558 ms 2024-03-30T13:24:21.223 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:21 smithi159 ceph-mon[24787]: pgmap v255: 97 pgs: 97 active+clean; 201 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 871 op/s 2024-03-30T13:24:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:21 smithi187 ceph-mon[27499]: pgmap v255: 97 pgs: 97 active+clean; 201 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 871 op/s 2024-03-30T13:24:21.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 28100 19.67 MB/sec warmup 43 sec latency 21.244 ms 2024-03-30T13:24:22.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 28662 19.69 MB/sec warmup 44 sec latency 33.993 ms 2024-03-30T13:24:23.236 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:23 smithi159 ceph-mon[24787]: pgmap v256: 97 pgs: 97 active+clean; 205 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.5 MiB/s wr, 846 op/s 2024-03-30T13:24:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:23 smithi187 ceph-mon[27499]: pgmap v256: 97 pgs: 97 active+clean; 205 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.5 MiB/s wr, 846 op/s 2024-03-30T13:24:23.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 29224 19.43 MB/sec warmup 45 sec latency 22.300 ms 2024-03-30T13:24:24.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:24 smithi159 ceph-mon[24787]: pgmap v257: 97 pgs: 97 active+clean; 205 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.7 MiB/s wr, 938 op/s 2024-03-30T13:24:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:24 smithi187 ceph-mon[27499]: pgmap v257: 97 pgs: 97 active+clean; 205 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.7 MiB/s wr, 938 op/s 2024-03-30T13:24:24.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 29913 19.34 MB/sec warmup 46 sec latency 27.253 ms 2024-03-30T13:24:25.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 30719 19.23 MB/sec warmup 47 sec latency 29.773 ms 2024-03-30T13:24:26.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 31600 19.59 MB/sec warmup 48 sec latency 25.435 ms 2024-03-30T13:24:27.263 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:27 smithi159 ceph-mon[24787]: pgmap v258: 97 pgs: 97 active+clean; 211 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 815 op/s 2024-03-30T13:24:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:27 smithi187 ceph-mon[27499]: pgmap v258: 97 pgs: 97 active+clean; 211 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 815 op/s 2024-03-30T13:24:27.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 32264 19.64 MB/sec warmup 49 sec latency 30.084 ms 2024-03-30T13:24:28.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 32871 19.41 MB/sec warmup 50 sec latency 17.763 ms 2024-03-30T13:24:29.218 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:29 smithi159 ceph-mon[24787]: pgmap v259: 97 pgs: 97 active+clean; 216 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.0 MiB/s wr, 805 op/s 2024-03-30T13:24:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:29 smithi187 ceph-mon[27499]: pgmap v259: 97 pgs: 97 active+clean; 216 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.0 MiB/s wr, 805 op/s 2024-03-30T13:24:29.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 33728 19.46 MB/sec warmup 51 sec latency 19.588 ms 2024-03-30T13:24:30.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 34562 19.46 MB/sec warmup 52 sec latency 21.118 ms 2024-03-30T13:24:31.227 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:31 smithi159 ceph-mon[24787]: pgmap v260: 97 pgs: 97 active+clean; 218 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.9 MiB/s wr, 902 op/s 2024-03-30T13:24:31.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:31 smithi187 ceph-mon[27499]: pgmap v260: 97 pgs: 97 active+clean; 218 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.9 MiB/s wr, 902 op/s 2024-03-30T13:24:31.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 35377 19.68 MB/sec warmup 53 sec latency 17.475 ms 2024-03-30T13:24:32.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 36003 19.65 MB/sec warmup 54 sec latency 21.606 ms 2024-03-30T13:24:33.240 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:33 smithi159 ceph-mon[24787]: pgmap v261: 97 pgs: 97 active+clean; 220 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 736 op/s 2024-03-30T13:24:33.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:33 smithi187 ceph-mon[27499]: pgmap v261: 97 pgs: 97 active+clean; 220 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 736 op/s 2024-03-30T13:24:33.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 36609 19.53 MB/sec warmup 55 sec latency 23.003 ms 2024-03-30T13:24:34.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:34 smithi159 ceph-mon[24787]: pgmap v262: 97 pgs: 97 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 952 op/s 2024-03-30T13:24:34.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:34 smithi187 ceph-mon[27499]: pgmap v262: 97 pgs: 97 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 952 op/s 2024-03-30T13:24:34.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 37241 19.38 MB/sec warmup 56 sec latency 29.181 ms 2024-03-30T13:24:35.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 37730 19.20 MB/sec warmup 57 sec latency 40.013 ms 2024-03-30T13:24:36.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 38433 19.28 MB/sec warmup 58 sec latency 18.708 ms 2024-03-30T13:24:37.263 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:37 smithi159 ceph-mon[24787]: pgmap v263: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 992 op/s 2024-03-30T13:24:37.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:37 smithi187 ceph-mon[27499]: pgmap v263: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 992 op/s 2024-03-30T13:24:37.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 39090 19.41 MB/sec warmup 59 sec latency 68.293 ms 2024-03-30T13:24:38.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 39612 19.28 MB/sec warmup 60 sec latency 45.842 ms 2024-03-30T13:24:39.305 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:39 smithi159 ceph-mon[24787]: pgmap v264: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 929 op/s 2024-03-30T13:24:39.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:39 smithi187 ceph-mon[27499]: pgmap v264: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 929 op/s 2024-03-30T13:24:39.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 40131 19.13 MB/sec warmup 61 sec latency 56.965 ms 2024-03-30T13:24:40.176 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:24:40.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 40741 18.99 MB/sec warmup 62 sec latency 33.240 ms 2024-03-30T13:24:41.181 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:24:41.181 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (5m) 64s ago 7m 23.0M - ba2b418f427c a3dca4f7bb16 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (7m) 64s ago 7m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (6m) 66s ago 6m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (5m) 64s ago 7m 52.9M - 8.3.5 dad864ee21e9 3e8ed3a7ad2a 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (93s) 64s ago 92s 14.9M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (90s) 66s ago 90s 14.3M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:9283 running (8m) 64s ago 8m 459M - 17.2.0 e1d6a67b021e 2fcdf09abf3a 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (6m) 66s ago 6m 412M - 17.2.0 e1d6a67b021e 6d26676209f5 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (8m) 64s ago 8m 44.7M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (6m) 66s ago 6m 38.0M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (75s) 64s ago 74s 49.0M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (69s) 66s ago 69s 50.8M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (7m) 64s ago 7m 19.0M - 1dbe0e931976 c7e53de4dadb 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (6m) 66s ago 6m 21.5M - 1dbe0e931976 71a2445fdf72 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (5m) 64s ago 5m 45.9M 3216M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:24:41.182 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (5m) 64s ago 5m 44.0M 3216M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:24:41.183 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (4m) 64s ago 4m 42.4M 3216M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:24:41.183 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (4m) 64s ago 4m 40.5M 3216M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:24:41.183 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (4m) 66s ago 4m 43.0M 3984M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:24:41.183 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (3m) 66s ago 3m 38.7M 3984M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:24:41.183 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (3m) 66s ago 3m 40.4M 3984M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:24:41.183 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (2m) 66s ago 2m 40.7M 3984M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:24:41.183 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (6m) 64s ago 6m 62.4M - 514e6a882f6e 9863c014034e 2024-03-30T13:24:41.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:41 smithi159 ceph-mon[24787]: pgmap v265: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 979 op/s 2024-03-30T13:24:41.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:41 smithi159 ceph-mon[24787]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:41.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:41 smithi187 ceph-mon[27499]: pgmap v265: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 979 op/s 2024-03-30T13:24:41.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:41 smithi187 ceph-mon[27499]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:41.811 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:24:41.811 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:24:41.811 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:24:41.811 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:24:41.811 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:24:41.812 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:24:41.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 41277 18.89 MB/sec warmup 63 sec latency 70.648 ms 2024-03-30T13:24:42.306 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:24:42.306 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb", 2024-03-30T13:24:42.306 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:24:42.306 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [], 2024-03-30T13:24:42.306 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "", 2024-03-30T13:24:42.306 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:4e50bbcdd1630c7760b708f3a5b4d38a121e66cb image" 2024-03-30T13:24:42.306 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:24:42.363 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:42 smithi159 ceph-mon[24787]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:42.363 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:42 smithi159 ceph-mon[24787]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:42.363 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:42 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/737290031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:24:42.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:42 smithi187 ceph-mon[27499]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:42.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:42 smithi187 ceph-mon[27499]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:42.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:42 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/737290031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:24:42.851 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:24:42.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 41933 18.92 MB/sec warmup 64 sec latency 63.867 ms 2024-03-30T13:24:43.409 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:43 smithi159 ceph-mon[24787]: pgmap v266: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 950 op/s 2024-03-30T13:24:43.409 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:43 smithi159 ceph-mon[24787]: from='client.24421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:43.409 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:43 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1916582933' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:24:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:43 smithi187 ceph-mon[27499]: pgmap v266: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 950 op/s 2024-03-30T13:24:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:43 smithi187 ceph-mon[27499]: from='client.24421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:24:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:43 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1916582933' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:24:43.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 42595 19.06 MB/sec warmup 65 sec latency 90.129 ms 2024-03-30T13:24:44.671 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:44 smithi159 ceph-mon[24787]: pgmap v267: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.9 MiB/s wr, 915 op/s 2024-03-30T13:24:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:44 smithi187 ceph-mon[27499]: pgmap v267: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.9 MiB/s wr, 915 op/s 2024-03-30T13:24:44.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 43058 18.97 MB/sec warmup 66 sec latency 33.238 ms 2024-03-30T13:24:45.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 43376 18.74 MB/sec warmup 67 sec latency 44.356 ms 2024-03-30T13:24:46.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:24:46.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:24:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:24:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:24:46.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 43712 18.57 MB/sec warmup 68 sec latency 40.461 ms 2024-03-30T13:24:47.315 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:47 smithi159 ceph-mon[24787]: pgmap v268: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.2 MiB/s wr, 763 op/s 2024-03-30T13:24:47.315 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:47.315 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:24:47.315 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:47 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:47 smithi187 ceph-mon[27499]: pgmap v268: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.2 MiB/s wr, 763 op/s 2024-03-30T13:24:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:24:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:47 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:24:47.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 44097 18.44 MB/sec warmup 69 sec latency 42.058 ms 2024-03-30T13:24:48.347 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:48 smithi159 ceph-mon[24787]: Upgrade: Target is version 19.0.0-1967-g4e50bbcd (unknown) 2024-03-30T13:24:48.347 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:48 smithi159 ceph-mon[24787]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d'] 2024-03-30T13:24:48.347 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:48 smithi159 ceph-mon[24787]: Upgrade: Need to upgrade myself (mgr.smithi159.fnasly) 2024-03-30T13:24:48.347 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:48 smithi159 ceph-mon[24787]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d on smithi187 2024-03-30T13:24:48.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:48 smithi187 ceph-mon[27499]: Upgrade: Target is version 19.0.0-1967-g4e50bbcd (unknown) 2024-03-30T13:24:48.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:48 smithi187 ceph-mon[27499]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d'] 2024-03-30T13:24:48.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:48 smithi187 ceph-mon[27499]: Upgrade: Need to upgrade myself (mgr.smithi159.fnasly) 2024-03-30T13:24:48.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:48 smithi187 ceph-mon[27499]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d on smithi187 2024-03-30T13:24:48.866 INFO:teuthology.orchestra.run.smithi159.stdout: 5 44582 18.31 MB/sec warmup 70 sec latency 51.352 ms 2024-03-30T13:24:49.393 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:49 smithi159 ceph-mon[24787]: pgmap v269: 97 pgs: 97 active+clean; 257 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 710 op/s 2024-03-30T13:24:49.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:49 smithi187 ceph-mon[27499]: pgmap v269: 97 pgs: 97 active+clean; 257 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 710 op/s 2024-03-30T13:24:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: 5 45086 18.19 MB/sec warmup 71 sec latency 39.366 ms 2024-03-30T13:24:50.867 INFO:teuthology.orchestra.run.smithi159.stdout: 5 45657 18.25 MB/sec warmup 72 sec latency 33.679 ms 2024-03-30T13:24:51.406 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:51 smithi159 ceph-mon[24787]: pgmap v270: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 806 op/s 2024-03-30T13:24:51.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:51 smithi187 ceph-mon[27499]: pgmap v270: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 806 op/s 2024-03-30T13:24:51.872 INFO:teuthology.orchestra.run.smithi159.stdout: 5 46483 18.43 MB/sec warmup 73 sec latency 30.114 ms 2024-03-30T13:24:52.872 INFO:teuthology.orchestra.run.smithi159.stdout: 5 47238 18.35 MB/sec warmup 74 sec latency 12.881 ms 2024-03-30T13:24:53.414 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:53 smithi159 ceph-mon[24787]: pgmap v271: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 689 op/s 2024-03-30T13:24:53.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:53 smithi187 ceph-mon[27499]: pgmap v271: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 689 op/s 2024-03-30T13:24:53.872 INFO:teuthology.orchestra.run.smithi159.stdout: 5 48161 18.36 MB/sec warmup 75 sec latency 18.705 ms 2024-03-30T13:24:54.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:54 smithi159 ceph-mon[24787]: pgmap v272: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 743 op/s 2024-03-30T13:24:54.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:54 smithi187 ceph-mon[27499]: pgmap v272: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 743 op/s 2024-03-30T13:24:54.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 49054 18.47 MB/sec warmup 76 sec latency 21.381 ms 2024-03-30T13:24:55.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 49977 18.71 MB/sec warmup 77 sec latency 17.738 ms 2024-03-30T13:24:56.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 50647 18.58 MB/sec warmup 78 sec latency 19.360 ms 2024-03-30T13:24:57.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:57 smithi159 ceph-mon[24787]: pgmap v273: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.8 MiB/s wr, 776 op/s 2024-03-30T13:24:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:57 smithi187 ceph-mon[27499]: pgmap v273: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.8 MiB/s wr, 776 op/s 2024-03-30T13:24:57.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 51546 18.61 MB/sec warmup 79 sec latency 21.497 ms 2024-03-30T13:24:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 52335 18.62 MB/sec warmup 80 sec latency 19.407 ms 2024-03-30T13:24:59.453 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:24:59 smithi159 ceph-mon[24787]: pgmap v274: 97 pgs: 97 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.9 MiB/s wr, 854 op/s 2024-03-30T13:24:59.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:24:59 smithi187 ceph-mon[27499]: pgmap v274: 97 pgs: 97 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.9 MiB/s wr, 854 op/s 2024-03-30T13:24:59.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 53222 18.84 MB/sec warmup 81 sec latency 24.380 ms 2024-03-30T13:25:00.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 53705 18.76 MB/sec warmup 82 sec latency 23.319 ms 2024-03-30T13:25:01.478 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:01 smithi159 ceph-mon[24787]: pgmap v275: 97 pgs: 97 active+clean; 291 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.00k op/s 2024-03-30T13:25:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:01 smithi187 ceph-mon[27499]: pgmap v275: 97 pgs: 97 active+clean; 291 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.00k op/s 2024-03-30T13:25:01.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 54183 18.61 MB/sec warmup 83 sec latency 55.369 ms 2024-03-30T13:25:02.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 54817 18.56 MB/sec warmup 84 sec latency 59.741 ms 2024-03-30T13:25:03.239 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:03 smithi159 ceph-mon[24787]: pgmap v276: 97 pgs: 97 active+clean; 294 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 904 op/s 2024-03-30T13:25:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:03 smithi187 ceph-mon[27499]: pgmap v276: 97 pgs: 97 active+clean; 294 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 904 op/s 2024-03-30T13:25:03.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 55642 18.51 MB/sec warmup 85 sec latency 24.995 ms 2024-03-30T13:25:04.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:04 smithi187 ceph-mon[27499]: pgmap v277: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-03-30T13:25:04.773 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:04 smithi159 ceph-mon[24787]: pgmap v277: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-03-30T13:25:04.873 INFO:teuthology.orchestra.run.smithi159.stdout: 5 56469 18.74 MB/sec warmup 86 sec latency 23.917 ms 2024-03-30T13:25:05.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 56881 18.67 MB/sec warmup 87 sec latency 99.277 ms 2024-03-30T13:25:06.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 57373 18.58 MB/sec warmup 88 sec latency 33.592 ms 2024-03-30T13:25:07.359 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:07 smithi159 ceph-mon[24787]: pgmap v278: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 932 op/s 2024-03-30T13:25:07.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:07 smithi187 ceph-mon[27499]: pgmap v278: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 932 op/s 2024-03-30T13:25:07.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 57819 18.44 MB/sec warmup 89 sec latency 53.334 ms 2024-03-30T13:25:08.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 58382 18.39 MB/sec warmup 90 sec latency 31.617 ms 2024-03-30T13:25:09.377 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:09 smithi159 ceph-mon[24787]: pgmap v279: 97 pgs: 97 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-03-30T13:25:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:09 smithi187 ceph-mon[27499]: pgmap v279: 97 pgs: 97 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-03-30T13:25:09.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 59050 18.33 MB/sec warmup 91 sec latency 34.391 ms 2024-03-30T13:25:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 59638 18.35 MB/sec warmup 92 sec latency 34.104 ms 2024-03-30T13:25:11.417 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:11 smithi159 ceph-mon[24787]: pgmap v280: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 881 op/s 2024-03-30T13:25:11.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:11 smithi187 ceph-mon[27499]: pgmap v280: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 881 op/s 2024-03-30T13:25:11.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 60379 18.48 MB/sec warmup 93 sec latency 29.447 ms 2024-03-30T13:25:12.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 60788 18.40 MB/sec warmup 94 sec latency 57.857 ms 2024-03-30T13:25:13.273 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:25:13.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:13 smithi159 ceph-mon[24787]: pgmap v281: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 692 op/s 2024-03-30T13:25:13.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:13 smithi187 ceph-mon[27499]: pgmap v281: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 692 op/s 2024-03-30T13:25:13.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 61185 18.26 MB/sec warmup 95 sec latency 28.991 ms 2024-03-30T13:25:14.135 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:25:14.135 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (6m) 97s ago 7m 23.0M - ba2b418f427c a3dca4f7bb16 2024-03-30T13:25:14.135 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (7m) 97s ago 7m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:25:14.135 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (7m) 99s ago 7m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:25:14.135 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (6m) 97s ago 7m 52.9M - 8.3.5 dad864ee21e9 3e8ed3a7ad2a 2024-03-30T13:25:14.135 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (2m) 97s ago 2m 14.9M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:25:14.135 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (2m) 99s ago 2m 14.3M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:9283 running (9m) 97s ago 9m 459M - 17.2.0 e1d6a67b021e 2fcdf09abf3a 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (7m) 99s ago 7m 412M - 17.2.0 e1d6a67b021e 6d26676209f5 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (9m) 97s ago 9m 44.7M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (7m) 99s ago 7m 38.0M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (108s) 97s ago 107s 49.0M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (102s) 99s ago 102s 50.8M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (7m) 97s ago 7m 19.0M - 1dbe0e931976 c7e53de4dadb 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (7m) 99s ago 6m 21.5M - 1dbe0e931976 71a2445fdf72 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (6m) 97s ago 6m 45.9M 3216M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (6m) 97s ago 5m 44.0M 3216M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (5m) 97s ago 5m 42.4M 3216M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (5m) 97s ago 5m 40.5M 3216M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (4m) 99s ago 4m 43.0M 3984M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (4m) 99s ago 4m 38.7M 3984M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:25:14.136 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (3m) 99s ago 3m 40.4M 3984M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:25:14.137 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (3m) 99s ago 3m 40.7M 3984M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:25:14.137 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (6m) 97s ago 6m 62.4M - 514e6a882f6e 9863c014034e 2024-03-30T13:25:14.459 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:14 smithi159 ceph-mon[24787]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:14.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:14 smithi187 ceph-mon[27499]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:14.665 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:25:14.665 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:25:14.665 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:25:14.666 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:25:14.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 61687 18.21 MB/sec warmup 96 sec latency 34.969 ms 2024-03-30T13:25:15.193 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:25:15.193 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:25:15.193 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:25:15.193 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [], 2024-03-30T13:25:15.194 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "0/23 daemons upgraded", 2024-03-30T13:25:15.194 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d image on host smithi187" 2024-03-30T13:25:15.194 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:25:15.468 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:15 smithi159 ceph-mon[24787]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:15.468 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:15 smithi159 ceph-mon[24787]: from='client.24437 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:15.468 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:15 smithi159 ceph-mon[24787]: pgmap v282: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 854 op/s 2024-03-30T13:25:15.468 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:15 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3850086287' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:25:15.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:15 smithi187 ceph-mon[27499]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:15.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:15 smithi187 ceph-mon[27499]: from='client.24437 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:15.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:15 smithi187 ceph-mon[27499]: pgmap v282: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 854 op/s 2024-03-30T13:25:15.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:15 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3850086287' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:25:15.695 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:25:15.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 62225 18.13 MB/sec warmup 97 sec latency 101.017 ms 2024-03-30T13:25:16.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:16 smithi159 ceph-mon[24787]: from='client.24443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:16.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:16 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3979162146' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:25:16.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:16 smithi187 ceph-mon[27499]: from='client.24443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:16.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:16 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3979162146' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:25:16.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 62647 18.00 MB/sec warmup 98 sec latency 37.277 ms 2024-03-30T13:25:17.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:17 smithi159 ceph-mon[24787]: pgmap v283: 97 pgs: 97 active+clean; 313 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.7 MiB/s wr, 700 op/s 2024-03-30T13:25:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:17 smithi187 ceph-mon[27499]: pgmap v283: 97 pgs: 97 active+clean; 313 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.7 MiB/s wr, 700 op/s 2024-03-30T13:25:17.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 63168 18.01 MB/sec warmup 99 sec latency 33.631 ms 2024-03-30T13:25:18.879 INFO:teuthology.orchestra.run.smithi159.stdout: 5 63722 18.09 MB/sec warmup 100 sec latency 32.217 ms 2024-03-30T13:25:19.295 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:19 smithi159 ceph-mon[24787]: pgmap v284: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.8 MiB/s wr, 912 op/s 2024-03-30T13:25:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:19 smithi187 ceph-mon[27499]: pgmap v284: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.8 MiB/s wr, 912 op/s 2024-03-30T13:25:19.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 64170 18.05 MB/sec warmup 101 sec latency 37.045 ms 2024-03-30T13:25:20.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 64576 17.92 MB/sec warmup 102 sec latency 25.062 ms 2024-03-30T13:25:21.309 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:21 smithi159 ceph-mon[24787]: pgmap v285: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 735 op/s 2024-03-30T13:25:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:21 smithi187 ceph-mon[27499]: pgmap v285: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 735 op/s 2024-03-30T13:25:21.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 65095 17.86 MB/sec warmup 103 sec latency 27.691 ms 2024-03-30T13:25:22.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 65732 17.83 MB/sec warmup 104 sec latency 33.022 ms 2024-03-30T13:25:23.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:23 smithi159 ceph-mon[24787]: pgmap v286: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.9 MiB/s wr, 685 op/s 2024-03-30T13:25:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:23 smithi187 ceph-mon[27499]: pgmap v286: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.9 MiB/s wr, 685 op/s 2024-03-30T13:25:23.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 66209 17.71 MB/sec warmup 105 sec latency 107.850 ms 2024-03-30T13:25:24.768 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:24 smithi159 ceph-mon[24787]: pgmap v287: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.3 MiB/s wr, 829 op/s 2024-03-30T13:25:24.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 66837 17.76 MB/sec warmup 106 sec latency 45.037 ms 2024-03-30T13:25:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:24 smithi187 ceph-mon[27499]: pgmap v287: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.3 MiB/s wr, 829 op/s 2024-03-30T13:25:25.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 67549 17.89 MB/sec warmup 107 sec latency 16.433 ms 2024-03-30T13:25:26.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 68075 17.80 MB/sec warmup 108 sec latency 19.696 ms 2024-03-30T13:25:27.289 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:27 smithi159 ceph-mon[24787]: pgmap v288: 97 pgs: 97 active+clean; 330 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 6.7 MiB/s wr, 670 op/s 2024-03-30T13:25:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:27 smithi187 ceph-mon[27499]: pgmap v288: 97 pgs: 97 active+clean; 330 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 6.7 MiB/s wr, 670 op/s 2024-03-30T13:25:27.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 68669 17.76 MB/sec warmup 109 sec latency 22.851 ms 2024-03-30T13:25:28.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 69382 17.73 MB/sec warmup 110 sec latency 36.493 ms 2024-03-30T13:25:29.305 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:29 smithi159 ceph-mon[24787]: pgmap v289: 97 pgs: 97 active+clean; 340 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.3 MiB/s wr, 916 op/s 2024-03-30T13:25:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:29 smithi187 ceph-mon[27499]: pgmap v289: 97 pgs: 97 active+clean; 340 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.3 MiB/s wr, 916 op/s 2024-03-30T13:25:29.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 70073 17.73 MB/sec warmup 111 sec latency 32.246 ms 2024-03-30T13:25:30.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 70956 17.90 MB/sec warmup 112 sec latency 20.672 ms 2024-03-30T13:25:31.313 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:31 smithi159 ceph-mon[24787]: pgmap v290: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 721 op/s 2024-03-30T13:25:31.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:31 smithi187 ceph-mon[27499]: pgmap v290: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 721 op/s 2024-03-30T13:25:31.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 71541 17.86 MB/sec warmup 113 sec latency 25.856 ms 2024-03-30T13:25:32.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 72188 17.81 MB/sec warmup 114 sec latency 16.369 ms 2024-03-30T13:25:33.324 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:33 smithi159 ceph-mon[24787]: pgmap v291: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.8 MiB/s wr, 856 op/s 2024-03-30T13:25:33.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:33 smithi187 ceph-mon[27499]: pgmap v291: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.8 MiB/s wr, 856 op/s 2024-03-30T13:25:33.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 72793 17.78 MB/sec warmup 115 sec latency 54.448 ms 2024-03-30T13:25:34.581 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:34 smithi159 ceph-mon[24787]: pgmap v292: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 9.4 MiB/s wr, 963 op/s 2024-03-30T13:25:34.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 73365 17.69 MB/sec warmup 116 sec latency 27.059 ms 2024-03-30T13:25:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:34 smithi187 ceph-mon[27499]: pgmap v292: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 9.4 MiB/s wr, 963 op/s 2024-03-30T13:25:35.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 73870 17.71 MB/sec warmup 117 sec latency 29.077 ms 2024-03-30T13:25:36.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 74356 17.74 MB/sec warmup 118 sec latency 38.653 ms 2024-03-30T13:25:37.372 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:37 smithi159 ceph-mon[24787]: pgmap v293: 97 pgs: 97 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 856 op/s 2024-03-30T13:25:37.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:37 smithi187 ceph-mon[27499]: pgmap v293: 97 pgs: 97 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 856 op/s 2024-03-30T13:25:37.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 74778 17.73 MB/sec warmup 119 sec latency 50.795 ms 2024-03-30T13:25:39.399 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:39 smithi159 ceph-mon[24787]: pgmap v294: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 9.5 MiB/s wr, 981 op/s 2024-03-30T13:25:39.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:39 smithi187 ceph-mon[27499]: pgmap v294: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 9.5 MiB/s wr, 981 op/s 2024-03-30T13:25:39.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 75855 14.44 MB/sec execute 1 sec latency 28.550 ms 2024-03-30T13:25:40.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 76478 13.63 MB/sec execute 2 sec latency 26.915 ms 2024-03-30T13:25:41.304 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:41 smithi187 ceph-mon[27499]: pgmap v295: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 891 op/s 2024-03-30T13:25:41.424 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:41 smithi159 ceph-mon[24787]: pgmap v295: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 891 op/s 2024-03-30T13:25:41.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 77024 12.45 MB/sec execute 3 sec latency 32.088 ms 2024-03-30T13:25:42.444 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:42 smithi159 ceph-mon[24787]: Upgrade: Updating mgr.smithi187.jpkaud 2024-03-30T13:25:42.444 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:25:42.444 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:25:42.444 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:25:42.445 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:42 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:25:42.445 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:42 smithi159 ceph-mon[24787]: Deploying daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:25:42.445 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:42 smithi159 ceph-mon[24787]: pgmap v296: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.7 MiB/s wr, 906 op/s 2024-03-30T13:25:42.739 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:42 smithi187 ceph-mon[27499]: Upgrade: Updating mgr.smithi187.jpkaud 2024-03-30T13:25:42.739 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:25:42.739 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:25:42.739 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:25:42.739 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:42 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:25:42.739 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:42 smithi187 ceph-mon[27499]: Deploying daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:25:42.739 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:42 smithi187 ceph-mon[27499]: pgmap v296: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.7 MiB/s wr, 906 op/s 2024-03-30T13:25:42.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 77642 15.67 MB/sec execute 4 sec latency 35.956 ms 2024-03-30T13:25:43.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 78212 17.30 MB/sec execute 5 sec latency 24.490 ms 2024-03-30T13:25:44.651 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:44 smithi159 ceph-mon[24787]: pgmap v297: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.3 MiB/s wr, 873 op/s 2024-03-30T13:25:44.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 78634 15.72 MB/sec execute 6 sec latency 68.277 ms 2024-03-30T13:25:44.903 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:44 smithi187 ceph-mon[27499]: pgmap v297: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.3 MiB/s wr, 873 op/s 2024-03-30T13:25:45.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 79120 14.30 MB/sec execute 7 sec latency 39.827 ms 2024-03-30T13:25:46.198 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:25:46.410 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:25:46.410 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:46 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:25:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:25:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:46 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:25:46.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 79624 14.23 MB/sec execute 8 sec latency 78.259 ms 2024-03-30T13:25:47.167 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (7m) 2m ago 8m 23.0M - ba2b418f427c a3dca4f7bb16 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (8m) 2m ago 8m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (7m) 2m ago 7m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (6m) 2m ago 8m 52.9M - 8.3.5 dad864ee21e9 3e8ed3a7ad2a 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (2m) 2m ago 2m 14.9M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (2m) 2m ago 2m 14.3M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:9283 running (9m) 2m ago 9m 459M - 17.2.0 e1d6a67b021e 2fcdf09abf3a 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (7m) 2m ago 7m 412M - 17.2.0 e1d6a67b021e 6d26676209f5 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (9m) 2m ago 9m 44.7M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (7m) 2m ago 7m 38.0M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (2m) 2m ago 2m 49.0M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (2m) 2m ago 2m 50.8M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (8m) 2m ago 8m 19.0M - 1dbe0e931976 c7e53de4dadb 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (7m) 2m ago 7m 21.5M - 1dbe0e931976 71a2445fdf72 2024-03-30T13:25:47.168 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (7m) 2m ago 6m 45.9M 3216M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:25:47.169 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (6m) 2m ago 6m 44.0M 3216M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:25:47.169 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (6m) 2m ago 6m 42.4M 3216M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:25:47.169 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (5m) 2m ago 5m 40.5M 3216M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:25:47.169 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (5m) 2m ago 5m 43.0M 3984M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:25:47.169 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (4m) 2m ago 4m 38.7M 3984M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:25:47.169 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (4m) 2m ago 4m 40.4M 3984M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:25:47.169 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (4m) 2m ago 4m 40.7M 3984M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:25:47.169 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (7m) 2m ago 7m 62.4M - 514e6a882f6e 9863c014034e 2024-03-30T13:25:47.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:47 smithi159 ceph-mon[24787]: pgmap v298: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.9 MiB/s wr, 798 op/s 2024-03-30T13:25:47.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:47 smithi159 ceph-mon[24787]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:47 smithi187 ceph-mon[27499]: pgmap v298: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.9 MiB/s wr, 798 op/s 2024-03-30T13:25:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:47 smithi187 ceph-mon[27499]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:47.709 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:25:47.709 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:25:47.709 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:25:47.709 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:25:47.709 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:25:47.709 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:25:47.709 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:25:47.710 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:25:47.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 80193 13.79 MB/sec execute 9 sec latency 87.160 ms 2024-03-30T13:25:48.194 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:25:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:25:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:25:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [], 2024-03-30T13:25:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "0/23 daemons upgraded", 2024-03-30T13:25:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading mgr daemons" 2024-03-30T13:25:48.195 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:25:48.458 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:48 smithi159 ceph-mon[24787]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:48.459 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:48 smithi159 ceph-mon[24787]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:48.459 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:48 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3373538226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:25:48.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:48 smithi187 ceph-mon[27499]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:48.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:48 smithi187 ceph-mon[27499]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:48.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:48 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3373538226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:25:48.823 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:25:48.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 80688 13.80 MB/sec execute 10 sec latency 28.632 ms 2024-03-30T13:25:49.329 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:49 smithi187 ceph-mon[27499]: pgmap v299: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.0 MiB/s wr, 769 op/s 2024-03-30T13:25:49.330 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:49 smithi187 ceph-mon[27499]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:49.330 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:49 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/44671081' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:25:49.470 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:49 smithi159 ceph-mon[24787]: pgmap v299: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.0 MiB/s wr, 769 op/s 2024-03-30T13:25:49.471 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:49 smithi159 ceph-mon[24787]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:25:49.471 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:49 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/44671081' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:25:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 81113 14.22 MB/sec execute 11 sec latency 44.541 ms 2024-03-30T13:25:50.485 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:50 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:25:50.486 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:50 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:25:50.486 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:50 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:25:50.486 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:50 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:25:50.486 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:50 smithi159 ceph-mon[24787]: pgmap v300: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.0 MiB/s wr, 738 op/s 2024-03-30T13:25:50.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:50 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:25:50.836 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:50 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:25:50.836 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:50 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:25:50.836 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:50 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:25:50.836 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:50 smithi187 ceph-mon[27499]: pgmap v300: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.0 MiB/s wr, 738 op/s 2024-03-30T13:25:50.880 INFO:teuthology.orchestra.run.smithi159.stdout: 5 81518 14.27 MB/sec execute 12 sec latency 99.872 ms 2024-03-30T13:25:51.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 81857 14.24 MB/sec execute 13 sec latency 78.048 ms 2024-03-30T13:25:52.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 82194 13.58 MB/sec execute 14 sec latency 24.824 ms 2024-03-30T13:25:53.289 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:53 smithi159 ceph-mon[24787]: pgmap v301: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 645 op/s 2024-03-30T13:25:53.412 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:53 smithi187 ceph-mon[27499]: pgmap v301: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 645 op/s 2024-03-30T13:25:53.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 82549 12.98 MB/sec execute 15 sec latency 105.514 ms 2024-03-30T13:25:54.568 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:54 smithi159 ceph-mon[24787]: pgmap v302: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 756 op/s 2024-03-30T13:25:54.612 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:54 smithi187 ceph-mon[27499]: pgmap v302: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 756 op/s 2024-03-30T13:25:54.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 82963 12.86 MB/sec execute 16 sec latency 29.696 ms 2024-03-30T13:25:55.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 83510 12.76 MB/sec execute 17 sec latency 42.112 ms 2024-03-30T13:25:56.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 84099 12.71 MB/sec execute 18 sec latency 45.848 ms 2024-03-30T13:25:57.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:57 smithi159 ceph-mon[24787]: pgmap v303: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 6.6 MiB/s wr, 614 op/s 2024-03-30T13:25:57.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:57 smithi159 ceph-mon[24787]: Standby manager daemon smithi187.jpkaud restarted 2024-03-30T13:25:57.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:57 smithi159 ceph-mon[24787]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:25:57.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:57 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:25:57.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:57 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:25:57.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:57 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:25:57.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:57 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:25:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:57 smithi187 ceph-mon[27499]: pgmap v303: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 6.6 MiB/s wr, 614 op/s 2024-03-30T13:25:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:57 smithi187 ceph-mon[27499]: Standby manager daemon smithi187.jpkaud restarted 2024-03-30T13:25:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:57 smithi187 ceph-mon[27499]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:25:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:57 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:25:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:57 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:25:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:57 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:25:57.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:57 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:25:57.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 84754 13.54 MB/sec execute 19 sec latency 33.837 ms 2024-03-30T13:25:58.374 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:58 smithi159 ceph-mon[24787]: mgrmap e22: smithi159.fnasly(active, since 8m), standbys: smithi187.jpkaud 2024-03-30T13:25:58.520 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:58 smithi187 ceph-mon[27499]: mgrmap e22: smithi159.fnasly(active, since 8m), standbys: smithi187.jpkaud 2024-03-30T13:25:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 85344 13.99 MB/sec execute 20 sec latency 60.447 ms 2024-03-30T13:25:59.378 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:59 smithi159 ceph-mon[24787]: pgmap v304: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.1 MiB/s wr, 713 op/s 2024-03-30T13:25:59.378 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:25:59.378 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:25:59 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:25:59.459 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:59 smithi187 ceph-mon[27499]: pgmap v304: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.1 MiB/s wr, 713 op/s 2024-03-30T13:25:59.459 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:59 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:25:59.459 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:25:59 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:25:59.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 85785 13.62 MB/sec execute 21 sec latency 40.633 ms 2024-03-30T13:26:00.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 86333 13.55 MB/sec execute 22 sec latency 35.374 ms 2024-03-30T13:26:01.395 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:01 smithi159 ceph-mon[24787]: pgmap v305: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 705 op/s 2024-03-30T13:26:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:01 smithi187 ceph-mon[27499]: pgmap v305: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 705 op/s 2024-03-30T13:26:01.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 86882 13.54 MB/sec execute 23 sec latency 35.643 ms 2024-03-30T13:26:02.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 87474 13.25 MB/sec execute 24 sec latency 31.399 ms 2024-03-30T13:26:03.424 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:03 smithi159 ceph-mon[24787]: pgmap v306: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 669 op/s 2024-03-30T13:26:03.424 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:03 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:26:03.424 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:03 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:26:03.424 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:03 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:26:03.425 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:03 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:26:03.425 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:03 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr fail", "who": "smithi159.fnasly"}]: dispatch 2024-03-30T13:26:03.425 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:03 smithi159 ceph-mon[24787]: osdmap e71: 8 total, 8 up, 8 in 2024-03-30T13:26:03.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:03 smithi187 ceph-mon[27499]: pgmap v306: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 669 op/s 2024-03-30T13:26:03.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:03 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:26:03.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:03 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:26:03.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:03 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' 2024-03-30T13:26:03.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:03 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:26:03.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:03 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr fail", "who": "smithi159.fnasly"}]: dispatch 2024-03-30T13:26:03.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:03 smithi187 ceph-mon[27499]: osdmap e71: 8 total, 8 up, 8 in 2024-03-30T13:26:03.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 88015 13.43 MB/sec execute 25 sec latency 41.705 ms 2024-03-30T13:26:04.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: Upgrade: Need to upgrade myself (mgr.smithi159.fnasly) 2024-03-30T13:26:04.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: Failing over to other MGR 2024-03-30T13:26:04.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "mgr fail", "who": "smithi159.fnasly"}]': finished 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: mgrmap e23: smithi187.jpkaud(active, starting, since 0.994349s) 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr metadata", "who": "smithi187.jpkaud", "id": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:26:04.435 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: Manager daemon smithi187.jpkaud is now available 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi187.jpkaud/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi187.jpkaud/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi187.jpkaud/trash_purge_schedule"}]: dispatch 2024-03-30T13:26:04.436 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:04 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi187.jpkaud/trash_purge_schedule"}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: Upgrade: Need to upgrade myself (mgr.smithi159.fnasly) 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: Failing over to other MGR 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.14202 172.21.15.159:0/1723843915' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "mgr fail", "who": "smithi159.fnasly"}]': finished 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: mgrmap e23: smithi187.jpkaud(active, starting, since 0.994349s) 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr metadata", "who": "smithi187.jpkaud", "id": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:26:04.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: Manager daemon smithi187.jpkaud is now available 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi187.jpkaud/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi187.jpkaud/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi187.jpkaud/trash_purge_schedule"}]: dispatch 2024-03-30T13:26:04.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:04 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi187.jpkaud/trash_purge_schedule"}]: dispatch 2024-03-30T13:26:04.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 88604 13.93 MB/sec execute 26 sec latency 28.784 ms 2024-03-30T13:26:05.440 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:05 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:05.441 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:05 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:05.441 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:05 smithi159 ceph-mon[24787]: Deploying cephadm binary to smithi187 2024-03-30T13:26:05.441 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:05 smithi159 ceph-mon[24787]: mgrmap e24: smithi187.jpkaud(active, since 2s) 2024-03-30T13:26:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:05 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:05 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:05 smithi187 ceph-mon[27499]: Deploying cephadm binary to smithi187 2024-03-30T13:26:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:05 smithi187 ceph-mon[27499]: mgrmap e24: smithi187.jpkaud(active, since 2s) 2024-03-30T13:26:05.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 89018 14.04 MB/sec execute 27 sec latency 35.747 ms 2024-03-30T13:26:06.454 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:06 smithi159 ceph-mon[24787]: Deploying cephadm binary to smithi159 2024-03-30T13:26:06.454 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:06 smithi159 ceph-mon[24787]: [30/Mar/2024:13:26:04] ENGINE Bus STARTING 2024-03-30T13:26:06.454 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:06 smithi159 ceph-mon[24787]: [30/Mar/2024:13:26:04] ENGINE Serving on http://172.21.15.187:8765 2024-03-30T13:26:06.454 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:06 smithi159 ceph-mon[24787]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-30T13:26:06.454 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:06 smithi159 ceph-mon[24787]: [30/Mar/2024:13:26:04] ENGINE Serving on https://172.21.15.187:7150 2024-03-30T13:26:06.454 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:06 smithi159 ceph-mon[24787]: [30/Mar/2024:13:26:04] ENGINE Bus STARTED 2024-03-30T13:26:06.454 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:06 smithi159 ceph-mon[24787]: mgrmap e25: smithi187.jpkaud(active, since 3s) 2024-03-30T13:26:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:06 smithi187 ceph-mon[27499]: Deploying cephadm binary to smithi159 2024-03-30T13:26:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:06 smithi187 ceph-mon[27499]: [30/Mar/2024:13:26:04] ENGINE Bus STARTING 2024-03-30T13:26:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:06 smithi187 ceph-mon[27499]: [30/Mar/2024:13:26:04] ENGINE Serving on http://172.21.15.187:8765 2024-03-30T13:26:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:06 smithi187 ceph-mon[27499]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-30T13:26:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:06 smithi187 ceph-mon[27499]: [30/Mar/2024:13:26:04] ENGINE Serving on https://172.21.15.187:7150 2024-03-30T13:26:06.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:06 smithi187 ceph-mon[27499]: [30/Mar/2024:13:26:04] ENGINE Bus STARTED 2024-03-30T13:26:06.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:06 smithi187 ceph-mon[27499]: mgrmap e25: smithi187.jpkaud(active, since 3s) 2024-03-30T13:26:06.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 89363 13.64 MB/sec execute 28 sec latency 22.277 ms 2024-03-30T13:26:07.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:07 smithi159 ceph-mon[24787]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-30T13:26:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:07 smithi187 ceph-mon[27499]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-30T13:26:07.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 89691 13.32 MB/sec execute 29 sec latency 45.508 ms 2024-03-30T13:26:08.488 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:08 smithi159 ceph-mon[24787]: mgrmap e26: smithi187.jpkaud(active, since 4s) 2024-03-30T13:26:08.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:08 smithi187 ceph-mon[27499]: mgrmap e26: smithi187.jpkaud(active, since 4s) 2024-03-30T13:26:08.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 90027 13.23 MB/sec execute 30 sec latency 53.046 ms 2024-03-30T13:26:09.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:09 smithi159 ceph-mon[24787]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-30T13:26:09.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:09 smithi187 ceph-mon[27499]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-30T13:26:09.881 INFO:teuthology.orchestra.run.smithi159.stdout: 5 90497 13.14 MB/sec execute 31 sec latency 35.587 ms 2024-03-30T13:26:10.511 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:10 smithi159 ceph-mon[24787]: pgmap v6: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-30T13:26:10.746 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:10 smithi187 ceph-mon[27499]: pgmap v6: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-30T13:26:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 90979 12.91 MB/sec execute 32 sec latency 38.037 ms 2024-03-30T13:26:11.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 91489 13.02 MB/sec execute 33 sec latency 56.862 ms 2024-03-30T13:26:12.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 92057 13.39 MB/sec execute 34 sec latency 25.677 ms 2024-03-30T13:26:13.039 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:12 smithi159 ceph-mon[24787]: pgmap v7: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.7 MiB/s wr, 437 op/s 2024-03-30T13:26:13.039 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:12 smithi159 ceph-mon[24787]: Standby manager daemon smithi159.fnasly started 2024-03-30T13:26:13.039 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:12 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.159:0/2661405282' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi159.fnasly/crt"}]: dispatch 2024-03-30T13:26:13.040 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:12 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.159:0/2661405282' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:26:13.040 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:12 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.159:0/2661405282' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi159.fnasly/key"}]: dispatch 2024-03-30T13:26:13.040 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:12 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.159:0/2661405282' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:26:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:12 smithi187 ceph-mon[27499]: pgmap v7: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.7 MiB/s wr, 437 op/s 2024-03-30T13:26:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:12 smithi187 ceph-mon[27499]: Standby manager daemon smithi159.fnasly started 2024-03-30T13:26:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:12 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.159:0/2661405282' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi159.fnasly/crt"}]: dispatch 2024-03-30T13:26:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:12 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.159:0/2661405282' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:26:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:12 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.159:0/2661405282' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi159.fnasly/key"}]: dispatch 2024-03-30T13:26:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:12 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.159:0/2661405282' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:26:13.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 92551 13.52 MB/sec execute 35 sec latency 33.522 ms 2024-03-30T13:26:14.050 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:13 smithi159 ceph-mon[24787]: mgrmap e27: smithi187.jpkaud(active, since 10s), standbys: smithi159.fnasly 2024-03-30T13:26:14.050 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:13 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:26:14.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:13 smithi187 ceph-mon[27499]: mgrmap e27: smithi187.jpkaud(active, since 10s), standbys: smithi159.fnasly 2024-03-30T13:26:14.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:13 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:26:14.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 92908 13.27 MB/sec execute 36 sec latency 34.026 ms 2024-03-30T13:26:15.061 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:14 smithi159 ceph-mon[24787]: pgmap v8: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 339 op/s 2024-03-30T13:26:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:14 smithi187 ceph-mon[27499]: pgmap v8: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 339 op/s 2024-03-30T13:26:15.883 INFO:teuthology.orchestra.run.smithi159.stdout: 5 93342 13.12 MB/sec execute 37 sec latency 30.633 ms 2024-03-30T13:26:16.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:15 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:16.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:15 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:16.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:15 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:16.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:15 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:16.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 93930 13.08 MB/sec execute 38 sec latency 33.039 ms 2024-03-30T13:26:17.353 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:17 smithi159 ceph-mon[24787]: pgmap v9: 97 pgs: 97 active+clean; 337 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.0 MiB/s wr, 586 op/s 2024-03-30T13:26:17.354 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:17 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:17.354 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:17 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:17 smithi187 ceph-mon[27499]: pgmap v9: 97 pgs: 97 active+clean; 337 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.0 MiB/s wr, 586 op/s 2024-03-30T13:26:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:17 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:17 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:17.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 94409 13.01 MB/sec execute 39 sec latency 37.722 ms 2024-03-30T13:26:18.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 94847 13.01 MB/sec execute 40 sec latency 48.661 ms 2024-03-30T13:26:19.297 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:26:19.443 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:19 smithi159 ceph-mon[24787]: pgmap v10: 97 pgs: 97 active+clean; 337 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.5 MiB/s wr, 533 op/s 2024-03-30T13:26:19.444 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:19 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:26:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:19 smithi187 ceph-mon[27499]: pgmap v10: 97 pgs: 97 active+clean; 337 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.5 MiB/s wr, 533 op/s 2024-03-30T13:26:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:19 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:26:19.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 95212 12.99 MB/sec execute 41 sec latency 39.692 ms 2024-03-30T13:26:20.355 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:26:20.355 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (7m) 0s ago 9m 28.5M - 0.23.0 ba2b418f427c a3dca4f7bb16 2024-03-30T13:26:20.355 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (9m) 0s ago 9m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:26:20.355 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (8m) 5s ago 8m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:26:20.355 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (7m) 0s ago 8m 52.2M - 8.3.5 dad864ee21e9 3e8ed3a7ad2a 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (3m) 0s ago 3m 37.0M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (3m) 5s ago 3m 15.3M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:9283 running (10m) 0s ago 10m 333M - 17.2.0 e1d6a67b021e 2fcdf09abf3a 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (35s) 5s ago 8m 489M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 11c6be81ba54 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (10m) 0s ago 10m 52.2M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (8m) 5s ago 8m 42.4M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (2m) 0s ago 2m 256M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (2m) 5s ago 2m 77.9M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (8m) 0s ago 8m 18.1M - 1.3.1 1dbe0e931976 c7e53de4dadb 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (8m) 5s ago 8m 22.3M - 1.3.1 1dbe0e931976 71a2445fdf72 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (7m) 0s ago 7m 364M 3216M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (7m) 0s ago 7m 280M 3216M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (6m) 0s ago 6m 246M 3216M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (6m) 0s ago 6m 323M 3216M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:26:20.356 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (5m) 5s ago 5m 329M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:26:20.357 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (5m) 5s ago 5m 356M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:26:20.357 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (5m) 5s ago 5m 296M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:26:20.357 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (4m) 5s ago 4m 409M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:26:20.357 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (7m) 0s ago 7m 64.6M - 2.33.4 514e6a882f6e 9863c014034e 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.457 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.457 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.457 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.457 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:20 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:20.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:20 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:20.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 95596 13.11 MB/sec execute 42 sec latency 39.259 ms 2024-03-30T13:26:20.932 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:26:20.932 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:26:20.932 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:26:20.933 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:26:20.934 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:26:21.455 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:26:21.455 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:26:21.455 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:26:21.455 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:26:21.455 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [], 2024-03-30T13:26:21.455 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "1/23 daemons upgraded", 2024-03-30T13:26:21.455 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "", 2024-03-30T13:26:21.456 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:26:21.456 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='client.14714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: pgmap v11: 97 pgs: 97 active+clean; 337 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.5 MiB/s wr, 533 op/s 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi187 to 2704M 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/587394423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.472 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:21.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:26:21.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:21 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:26:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='client.14714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: pgmap v11: 97 pgs: 97 active+clean; 337 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.5 MiB/s wr, 533 op/s 2024-03-30T13:26:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi187 to 2704M 2024-03-30T13:26:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/587394423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:26:21.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:21 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:26:21.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 95973 13.18 MB/sec execute 43 sec latency 34.490 ms 2024-03-30T13:26:22.026 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:26:22.480 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:22 smithi159 ceph-mon[24787]: Adjusting osd_memory_target on smithi159 to 1936M 2024-03-30T13:26:22.480 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:22 smithi159 ceph-mon[24787]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:26:22.480 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:22 smithi159 ceph-mon[24787]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-30T13:26:22.480 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:22 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/123937326' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:26:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:22 smithi187 ceph-mon[27499]: Adjusting osd_memory_target on smithi159 to 1936M 2024-03-30T13:26:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:22 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:26:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:22 smithi187 ceph-mon[27499]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-30T13:26:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:22 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/123937326' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:26:22.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 96255 12.98 MB/sec execute 44 sec latency 48.380 ms 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 8.1 MiB/s wr, 811 op/s 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.490 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:23 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:26:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:26:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:26:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 8.1 MiB/s wr, 811 op/s 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:23 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:23.882 INFO:teuthology.orchestra.run.smithi159.stdout: 5 96596 12.78 MB/sec execute 45 sec latency 30.069 ms 2024-03-30T13:26:24.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:24 smithi159 ceph-mon[24787]: Reconfiguring alertmanager.smithi159 (dependencies changed)... 2024-03-30T13:26:24.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:24 smithi159 ceph-mon[24787]: Reconfiguring daemon alertmanager.smithi159 on smithi159 2024-03-30T13:26:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:24 smithi187 ceph-mon[27499]: Reconfiguring alertmanager.smithi159 (dependencies changed)... 2024-03-30T13:26:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:24 smithi187 ceph-mon[27499]: Reconfiguring daemon alertmanager.smithi159 on smithi159 2024-03-30T13:26:24.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 96988 12.72 MB/sec execute 46 sec latency 49.085 ms 2024-03-30T13:26:25.513 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:25 smithi159 ceph-mon[24787]: pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 559 op/s 2024-03-30T13:26:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:25 smithi187 ceph-mon[27499]: pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 559 op/s 2024-03-30T13:26:25.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 97501 12.61 MB/sec execute 47 sec latency 32.555 ms 2024-03-30T13:26:26.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 98000 12.57 MB/sec execute 48 sec latency 36.685 ms 2024-03-30T13:26:27.536 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:27 smithi159 ceph-mon[24787]: pgmap v14: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 777 op/s 2024-03-30T13:26:27.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:27 smithi187 ceph-mon[27499]: pgmap v14: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 777 op/s 2024-03-30T13:26:27.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 98448 12.62 MB/sec execute 49 sec latency 61.770 ms 2024-03-30T13:26:28.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 98969 12.83 MB/sec execute 50 sec latency 31.972 ms 2024-03-30T13:26:29.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:29 smithi159 ceph-mon[24787]: pgmap v15: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 495 op/s 2024-03-30T13:26:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:29 smithi187 ceph-mon[27499]: pgmap v15: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 495 op/s 2024-03-30T13:26:29.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 99392 12.86 MB/sec execute 51 sec latency 28.181 ms 2024-03-30T13:26:30.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 99695 12.80 MB/sec execute 52 sec latency 35.214 ms 2024-03-30T13:26:31.362 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:31 smithi159 ceph-mon[24787]: pgmap v16: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 495 op/s 2024-03-30T13:26:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:31 smithi187 ceph-mon[27499]: pgmap v16: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 495 op/s 2024-03-30T13:26:31.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 99966 12.62 MB/sec execute 53 sec latency 41.597 ms 2024-03-30T13:26:32.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 100307 12.46 MB/sec execute 54 sec latency 36.402 ms 2024-03-30T13:26:33.459 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:33 smithi159 ceph-mon[24787]: pgmap v17: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-03-30T13:26:33.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:33 smithi187 ceph-mon[27499]: pgmap v17: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-03-30T13:26:33.884 INFO:teuthology.orchestra.run.smithi159.stdout: 5 100633 12.39 MB/sec execute 55 sec latency 49.073 ms 2024-03-30T13:26:34.730 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:34 smithi159 ceph-mon[24787]: pgmap v18: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 496 op/s 2024-03-30T13:26:34.731 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:26:34.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 101067 12.30 MB/sec execute 56 sec latency 50.921 ms 2024-03-30T13:26:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:34 smithi187 ceph-mon[27499]: pgmap v18: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 496 op/s 2024-03-30T13:26:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:26:35.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 101426 12.24 MB/sec execute 57 sec latency 46.537 ms 2024-03-30T13:26:36.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 101898 12.24 MB/sec execute 58 sec latency 31.342 ms 2024-03-30T13:26:37.023 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:36 smithi159 ceph-mon[24787]: pgmap v19: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 718 op/s 2024-03-30T13:26:37.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:36 smithi187 ceph-mon[27499]: pgmap v19: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 718 op/s 2024-03-30T13:26:37.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 102472 12.44 MB/sec execute 59 sec latency 19.684 ms 2024-03-30T13:26:38.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 102873 12.50 MB/sec execute 60 sec latency 39.973 ms 2024-03-30T13:26:39.069 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: pgmap v20: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 500 op/s 2024-03-30T13:26:39.069 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:39.069 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:39.069 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: Reconfiguring grafana.smithi159 (dependencies changed)... 2024-03-30T13:26:39.069 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: Regenerating cephadm self-signed grafana TLS certificates 2024-03-30T13:26:39.069 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:39.069 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:39.069 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-30T13:26:39.070 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-30T13:26:39.070 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:38 smithi159 ceph-mon[24787]: Reconfiguring daemon grafana.smithi159 on smithi159 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: pgmap v20: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 500 op/s 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: Reconfiguring grafana.smithi159 (dependencies changed)... 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: Regenerating cephadm self-signed grafana TLS certificates 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-30T13:26:39.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:38 smithi187 ceph-mon[27499]: Reconfiguring daemon grafana.smithi159 on smithi159 2024-03-30T13:26:39.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 103187 12.46 MB/sec execute 61 sec latency 42.281 ms 2024-03-30T13:26:40.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 103612 12.34 MB/sec execute 62 sec latency 27.607 ms 2024-03-30T13:26:41.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:40 smithi159 ceph-mon[24787]: pgmap v21: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 500 op/s 2024-03-30T13:26:41.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:40 smithi187 ceph-mon[27499]: pgmap v21: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 500 op/s 2024-03-30T13:26:41.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 104070 12.32 MB/sec execute 63 sec latency 32.819 ms 2024-03-30T13:26:42.848 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:42 smithi159 ceph-mon[24787]: pgmap v22: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 740 op/s 2024-03-30T13:26:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 104503 12.24 MB/sec execute 64 sec latency 60.320 ms 2024-03-30T13:26:43.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:42 smithi187 ceph-mon[27499]: pgmap v22: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 740 op/s 2024-03-30T13:26:43.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 104887 12.20 MB/sec execute 65 sec latency 53.901 ms 2024-03-30T13:26:44.610 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:44 smithi159 ceph-mon[24787]: pgmap v23: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-03-30T13:26:44.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 105438 12.20 MB/sec execute 66 sec latency 31.119 ms 2024-03-30T13:26:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:44 smithi187 ceph-mon[27499]: pgmap v23: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-03-30T13:26:45.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 105845 12.24 MB/sec execute 67 sec latency 38.321 ms 2024-03-30T13:26:46.885 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:46 smithi159 ceph-mon[24787]: pgmap v24: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 735 op/s 2024-03-30T13:26:46.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 106251 12.32 MB/sec execute 68 sec latency 40.648 ms 2024-03-30T13:26:47.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:46 smithi187 ceph-mon[27499]: pgmap v24: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 735 op/s 2024-03-30T13:26:47.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 106602 12.35 MB/sec execute 69 sec latency 48.727 ms 2024-03-30T13:26:48.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 106872 12.25 MB/sec execute 70 sec latency 48.308 ms 2024-03-30T13:26:48.914 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:48 smithi159 ceph-mon[24787]: pgmap v25: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-03-30T13:26:49.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:48 smithi187 ceph-mon[27499]: pgmap v25: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-03-30T13:26:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 107147 12.12 MB/sec execute 71 sec latency 42.277 ms 2024-03-30T13:26:49.927 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:49 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:26:50.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:49 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:26:50.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 107684 12.12 MB/sec execute 72 sec latency 30.167 ms 2024-03-30T13:26:50.932 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:50 smithi159 ceph-mon[24787]: pgmap v26: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-03-30T13:26:51.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:50 smithi187 ceph-mon[27499]: pgmap v26: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-03-30T13:26:51.886 INFO:teuthology.orchestra.run.smithi159.stdout: 5 108418 12.17 MB/sec execute 73 sec latency 31.599 ms 2024-03-30T13:26:52.615 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:26:52.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 109221 12.30 MB/sec execute 74 sec latency 18.311 ms 2024-03-30T13:26:52.959 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:52 smithi159 ceph-mon[24787]: pgmap v27: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.8 MiB/s wr, 729 op/s 2024-03-30T13:26:53.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:52 smithi187 ceph-mon[27499]: pgmap v27: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.8 MiB/s wr, 729 op/s 2024-03-30T13:26:53.762 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 starting - - - - 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (9m) 34s ago 9m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (8m) 38s ago 8m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (7m) 34s ago 9m 52.2M - 8.3.5 dad864ee21e9 3e8ed3a7ad2a 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (3m) 34s ago 3m 37.0M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (3m) 38s ago 3m 15.3M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:9283 running (10m) 34s ago 10m 333M - 17.2.0 e1d6a67b021e 2fcdf09abf3a 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (68s) 38s ago 8m 489M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 11c6be81ba54 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (10m) 34s ago 10m 52.2M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (8m) 38s ago 8m 42.4M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:26:53.763 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (3m) 34s ago 3m 256M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (3m) 38s ago 3m 77.9M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (9m) 34s ago 9m 18.1M - 1.3.1 1dbe0e931976 c7e53de4dadb 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (8m) 38s ago 8m 22.3M - 1.3.1 1dbe0e931976 71a2445fdf72 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (8m) 34s ago 8m 364M 1936M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (7m) 34s ago 7m 280M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (7m) 34s ago 7m 246M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (6m) 34s ago 6m 323M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (6m) 38s ago 6m 329M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (5m) 38s ago 5m 356M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (5m) 38s ago 5m 296M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:26:53.764 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (5m) 38s ago 5m 409M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:26:53.765 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (8m) 34s ago 8m 64.6M - 2.33.4 514e6a882f6e 9863c014034e 2024-03-30T13:26:53.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 110034 12.55 MB/sec execute 75 sec latency 31.773 ms 2024-03-30T13:26:53.966 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:53 smithi159 ceph-mon[24787]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:53.967 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:53 smithi159 ceph-mon[24787]: from='client.14738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:54.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:53 smithi187 ceph-mon[27499]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:54.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:53 smithi187 ceph-mon[27499]: from='client.14738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:54.478 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:26:54.478 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:26:54.478 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:26:54.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:26:54.478 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:26:54.478 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-30T13:26:54.478 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:26:54.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:26:54.479 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:26:54.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 110478 12.55 MB/sec execute 76 sec latency 31.340 ms 2024-03-30T13:26:54.982 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:54 smithi159 ceph-mon[24787]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:54.982 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:54 smithi159 ceph-mon[24787]: pgmap v28: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 489 op/s 2024-03-30T13:26:54.982 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:54 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1634398068' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [], 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "1/23 daemons upgraded", 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "", 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:26:55.003 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:26:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:54 smithi187 ceph-mon[27499]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:54 smithi187 ceph-mon[27499]: pgmap v28: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 489 op/s 2024-03-30T13:26:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:54 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1634398068' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:26:55.623 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:26:55.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 110827 12.45 MB/sec execute 77 sec latency 49.519 ms 2024-03-30T13:26:56.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:55 smithi159 ceph-mon[24787]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:56.007 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:55 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1757667504' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:26:56.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:55 smithi187 ceph-mon[27499]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:26:56.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:55 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1757667504' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:26:56.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 111215 12.41 MB/sec execute 78 sec latency 38.561 ms 2024-03-30T13:26:57.013 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:56 smithi159 ceph-mon[24787]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 870 op/s 2024-03-30T13:26:57.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:56 smithi187 ceph-mon[27499]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 870 op/s 2024-03-30T13:26:57.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 111607 12.34 MB/sec execute 79 sec latency 51.969 ms 2024-03-30T13:26:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 111978 12.27 MB/sec execute 80 sec latency 59.648 ms 2024-03-30T13:26:59.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:26:58 smithi159 ceph-mon[24787]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 597 op/s 2024-03-30T13:26:59.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:26:58 smithi187 ceph-mon[27499]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 597 op/s 2024-03-30T13:26:59.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 112325 12.20 MB/sec execute 81 sec latency 39.934 ms 2024-03-30T13:27:00.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:00 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:00.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:00 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:00.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:00 smithi159 ceph-mon[24787]: Reconfiguring node-exporter.smithi159 (dependencies changed)... 2024-03-30T13:27:00.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:00 smithi159 ceph-mon[24787]: Reconfiguring daemon node-exporter.smithi159 on smithi159 2024-03-30T13:27:00.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:00 smithi159 ceph-mon[24787]: pgmap v31: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 597 op/s 2024-03-30T13:27:00.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 112757 12.26 MB/sec execute 82 sec latency 33.362 ms 2024-03-30T13:27:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:00 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:00 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:00 smithi187 ceph-mon[27499]: Reconfiguring node-exporter.smithi159 (dependencies changed)... 2024-03-30T13:27:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:00 smithi187 ceph-mon[27499]: Reconfiguring daemon node-exporter.smithi159 on smithi159 2024-03-30T13:27:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:00 smithi187 ceph-mon[27499]: pgmap v31: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 597 op/s 2024-03-30T13:27:01.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 113191 12.33 MB/sec execute 83 sec latency 36.130 ms 2024-03-30T13:27:02.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 113569 12.35 MB/sec execute 84 sec latency 40.397 ms 2024-03-30T13:27:03.082 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:02 smithi159 ceph-mon[24787]: pgmap v32: 97 pgs: 97 active+clean; 323 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 830 op/s 2024-03-30T13:27:03.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:02 smithi187 ceph-mon[27499]: pgmap v32: 97 pgs: 97 active+clean; 323 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 830 op/s 2024-03-30T13:27:03.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 113973 12.33 MB/sec execute 85 sec latency 47.145 ms 2024-03-30T13:27:04.604 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:04 smithi159 ceph-mon[24787]: pgmap v33: 97 pgs: 97 active+clean; 323 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.9 MiB/s wr, 614 op/s 2024-03-30T13:27:04.605 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:04 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:27:04.605 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:04 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:04.605 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:04 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:04.605 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:04 smithi159 ceph-mon[24787]: Reconfiguring prometheus.smithi159 (dependencies changed)... 2024-03-30T13:27:04.605 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:04 smithi159 ceph-mon[24787]: Reconfiguring daemon prometheus.smithi159 on smithi159 2024-03-30T13:27:04.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 114262 12.23 MB/sec execute 86 sec latency 42.819 ms 2024-03-30T13:27:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:04 smithi187 ceph-mon[27499]: pgmap v33: 97 pgs: 97 active+clean; 323 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.9 MiB/s wr, 614 op/s 2024-03-30T13:27:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:04 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:27:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:04 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:04 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:04 smithi187 ceph-mon[27499]: Reconfiguring prometheus.smithi159 (dependencies changed)... 2024-03-30T13:27:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:04 smithi187 ceph-mon[27499]: Reconfiguring daemon prometheus.smithi159 on smithi159 2024-03-30T13:27:05.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 114659 12.21 MB/sec execute 87 sec latency 49.848 ms 2024-03-30T13:27:06.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:06 smithi159 ceph-mon[24787]: pgmap v34: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 845 op/s 2024-03-30T13:27:06.887 INFO:teuthology.orchestra.run.smithi159.stdout: 5 115041 12.14 MB/sec execute 88 sec latency 83.536 ms 2024-03-30T13:27:07.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:06 smithi187 ceph-mon[27499]: pgmap v34: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 845 op/s 2024-03-30T13:27:07.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 115464 12.10 MB/sec execute 89 sec latency 35.609 ms 2024-03-30T13:27:08.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 115977 12.13 MB/sec execute 90 sec latency 107.059 ms 2024-03-30T13:27:08.890 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:08 smithi159 ceph-mon[24787]: pgmap v35: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-03-30T13:27:09.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:08 smithi187 ceph-mon[27499]: pgmap v35: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-03-30T13:27:09.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 116594 12.24 MB/sec execute 91 sec latency 23.857 ms 2024-03-30T13:27:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 117022 12.29 MB/sec execute 92 sec latency 36.619 ms 2024-03-30T13:27:10.907 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:10 smithi159 ceph-mon[24787]: pgmap v36: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-03-30T13:27:11.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:10 smithi187 ceph-mon[27499]: pgmap v36: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-03-30T13:27:11.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 117472 12.29 MB/sec execute 93 sec latency 28.797 ms 2024-03-30T13:27:12.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 117812 12.22 MB/sec execute 94 sec latency 51.323 ms 2024-03-30T13:27:12.924 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:12 smithi159 ceph-mon[24787]: pgmap v37: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 738 op/s 2024-03-30T13:27:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:12 smithi187 ceph-mon[27499]: pgmap v37: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 738 op/s 2024-03-30T13:27:13.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 118271 12.20 MB/sec execute 95 sec latency 32.591 ms 2024-03-30T13:27:14.685 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:14 smithi159 ceph-mon[24787]: pgmap v38: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.4 MiB/s wr, 505 op/s 2024-03-30T13:27:14.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 118782 12.15 MB/sec execute 96 sec latency 49.243 ms 2024-03-30T13:27:15.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:14 smithi187 ceph-mon[27499]: pgmap v38: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.4 MiB/s wr, 505 op/s 2024-03-30T13:27:15.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 119255 12.15 MB/sec execute 97 sec latency 56.450 ms 2024-03-30T13:27:16.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 119853 12.22 MB/sec execute 98 sec latency 32.629 ms 2024-03-30T13:27:16.962 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:16 smithi159 ceph-mon[24787]: pgmap v39: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 775 op/s 2024-03-30T13:27:17.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:16 smithi187 ceph-mon[27499]: pgmap v39: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 775 op/s 2024-03-30T13:27:17.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 120518 12.37 MB/sec execute 99 sec latency 25.759 ms 2024-03-30T13:27:18.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 121273 12.43 MB/sec execute 100 sec latency 24.904 ms 2024-03-30T13:27:18.992 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:18 smithi159 ceph-mon[24787]: pgmap v40: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 544 op/s 2024-03-30T13:27:19.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:18 smithi187 ceph-mon[27499]: pgmap v40: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 544 op/s 2024-03-30T13:27:19.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 121926 12.45 MB/sec execute 101 sec latency 142.797 ms 2024-03-30T13:27:20.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:19 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:27:20.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:19 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:27:20.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 122555 12.43 MB/sec execute 102 sec latency 46.726 ms 2024-03-30T13:27:21.119 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:20 smithi159 ceph-mon[24787]: pgmap v41: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 544 op/s 2024-03-30T13:27:21.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:20 smithi187 ceph-mon[27499]: pgmap v41: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 544 op/s 2024-03-30T13:27:21.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 123029 12.43 MB/sec execute 103 sec latency 49.496 ms 2024-03-30T13:27:22.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 123567 12.47 MB/sec execute 104 sec latency 30.990 ms 2024-03-30T13:27:23.115 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:22 smithi159 ceph-mon[24787]: pgmap v42: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 913 op/s 2024-03-30T13:27:23.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:22 smithi187 ceph-mon[27499]: pgmap v42: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 913 op/s 2024-03-30T13:27:23.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 124258 12.62 MB/sec execute 105 sec latency 26.990 ms 2024-03-30T13:27:24.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:24 smithi159 ceph-mon[24787]: pgmap v43: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 639 op/s 2024-03-30T13:27:24.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 124800 12.62 MB/sec execute 106 sec latency 27.074 ms 2024-03-30T13:27:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:24 smithi187 ceph-mon[27499]: pgmap v43: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 639 op/s 2024-03-30T13:27:25.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 125114 12.55 MB/sec execute 107 sec latency 26.895 ms 2024-03-30T13:27:26.091 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:27:26.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 125510 12.53 MB/sec execute 108 sec latency 65.568 ms 2024-03-30T13:27:26.912 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:26 smithi159 ceph-mon[24787]: pgmap v44: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 968 op/s 2024-03-30T13:27:26.912 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:26 smithi159 ceph-mon[24787]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 starting - - - - 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (10m) 67s ago 10m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (9m) 72s ago 9m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 starting - - - - 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (4m) 67s ago 4m 37.0M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (4m) 72s ago 4m 15.3M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:9283 running (11m) 67s ago 11m 333M - 17.2.0 e1d6a67b021e 2fcdf09abf3a 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (102s) 72s ago 9m 489M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 11c6be81ba54 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (11m) 67s ago 11m 52.2M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:27:27.124 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (9m) 72s ago 9m 42.4M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (4m) 67s ago 4m 256M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (3m) 72s ago 3m 77.9M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 starting - - - - 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (9m) 72s ago 9m 22.3M - 1.3.1 1dbe0e931976 71a2445fdf72 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (8m) 67s ago 8m 364M 1936M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (8m) 67s ago 8m 280M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (7m) 67s ago 7m 246M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (7m) 67s ago 7m 323M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (6m) 72s ago 6m 329M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (6m) 72s ago 6m 356M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (6m) 72s ago 6m 296M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (5m) 72s ago 5m 409M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:27:27.125 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (8m) 67s ago 8m 64.6M - 2.33.4 514e6a882f6e 9863c014034e 2024-03-30T13:27:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:26 smithi187 ceph-mon[27499]: pgmap v44: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 968 op/s 2024-03-30T13:27:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:26 smithi187 ceph-mon[27499]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:27:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:27:27.800 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:27:27.800 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:27:27.800 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:27:27.800 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:27:27.800 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-30T13:27:27.800 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:27:27.800 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:27:27.800 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:27:27.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 126119 12.50 MB/sec execute 109 sec latency 44.061 ms 2024-03-30T13:27:27.915 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:27 smithi159 ceph-mon[24787]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:27:27.916 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:27 smithi159 ceph-mon[24787]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:27:27.916 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:27 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1857877261' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:27:28.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:27 smithi187 ceph-mon[27499]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:27:28.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:27 smithi187 ceph-mon[27499]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:27:28.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:27 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1857877261' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:27:28.357 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:27:28.357 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:27:28.357 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:27:28.357 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:27:28.357 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [], 2024-03-30T13:27:28.357 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "1/23 daemons upgraded", 2024-03-30T13:27:28.357 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "", 2024-03-30T13:27:28.358 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:27:28.358 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:27:28.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 126846 12.60 MB/sec execute 110 sec latency 31.557 ms 2024-03-30T13:27:28.920 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:28 smithi159 ceph-mon[24787]: pgmap v45: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.6 MiB/s wr, 698 op/s 2024-03-30T13:27:28.920 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:28 smithi159 ceph-mon[24787]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:27:29.054 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:27:29.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:28 smithi187 ceph-mon[27499]: pgmap v45: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.6 MiB/s wr, 698 op/s 2024-03-30T13:27:29.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:28 smithi187 ceph-mon[27499]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:27:29.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 127378 12.67 MB/sec execute 111 sec latency 27.228 ms 2024-03-30T13:27:29.933 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:29 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/4060247916' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:27:30.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:29 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/4060247916' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:27:30.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 127752 12.67 MB/sec execute 112 sec latency 43.329 ms 2024-03-30T13:27:30.955 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:30 smithi159 ceph-mon[24787]: pgmap v46: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.6 MiB/s wr, 698 op/s 2024-03-30T13:27:31.172 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:30 smithi187 ceph-mon[27499]: pgmap v46: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.6 MiB/s wr, 698 op/s 2024-03-30T13:27:31.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 128096 12.65 MB/sec execute 113 sec latency 42.419 ms 2024-03-30T13:27:32.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 128349 12.58 MB/sec execute 114 sec latency 69.369 ms 2024-03-30T13:27:32.981 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:32 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:32.982 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:32 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:32.982 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:32 smithi159 ceph-mon[24787]: Reconfiguring node-exporter.smithi187 (dependencies changed)... 2024-03-30T13:27:32.982 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:32 smithi159 ceph-mon[24787]: Reconfiguring daemon node-exporter.smithi187 on smithi187 2024-03-30T13:27:32.982 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:32 smithi159 ceph-mon[24787]: pgmap v47: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-03-30T13:27:33.010 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:32 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:33.010 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:32 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:33.010 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:32 smithi187 ceph-mon[27499]: Reconfiguring node-exporter.smithi187 (dependencies changed)... 2024-03-30T13:27:33.010 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:32 smithi187 ceph-mon[27499]: Reconfiguring daemon node-exporter.smithi187 on smithi187 2024-03-30T13:27:33.010 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:32 smithi187 ceph-mon[27499]: pgmap v47: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-03-30T13:27:33.888 INFO:teuthology.orchestra.run.smithi159.stdout: 5 128586 12.53 MB/sec execute 115 sec latency 44.837 ms 2024-03-30T13:27:34.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 128865 12.46 MB/sec execute 116 sec latency 75.227 ms 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: pgmap v48: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 640 op/s 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi159.front.sepia.ceph.com:9093"}]: dispatch 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi159.front.sepia.ceph.com:9093"}]: dispatch 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.009 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: Upgrade: Need to upgrade myself (mgr.smithi187.jpkaud) 2024-03-30T13:27:35.010 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:34 smithi159 ceph-mon[24787]: Upgrade: Need to upgrade myself (mgr.smithi187.jpkaud) 2024-03-30T13:27:35.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: pgmap v48: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 640 op/s 2024-03-30T13:27:35.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi159.front.sepia.ceph.com:9093"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi159.front.sepia.ceph.com:9093"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-03-30T13:27:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:27:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: Upgrade: Need to upgrade myself (mgr.smithi187.jpkaud) 2024-03-30T13:27:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:34 smithi187 ceph-mon[27499]: Upgrade: Need to upgrade myself (mgr.smithi187.jpkaud) 2024-03-30T13:27:35.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 129225 12.40 MB/sec execute 117 sec latency 57.438 ms 2024-03-30T13:27:36.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 129588 12.34 MB/sec execute 118 sec latency 38.639 ms 2024-03-30T13:27:37.053 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:36 smithi159 ceph-mon[24787]: pgmap v49: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.6 MiB/s wr, 833 op/s 2024-03-30T13:27:37.054 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:36 smithi159 ceph-mon[24787]: mgrmap e28: smithi187.jpkaud(active, since 93s), standbys: smithi159.fnasly 2024-03-30T13:27:37.054 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:36 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:37.054 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:36 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi159.fnasly", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:27:37.054 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:36 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi159.fnasly", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:27:37.054 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:36 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:27:37.054 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:36 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:27:37.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:36 smithi187 ceph-mon[27499]: pgmap v49: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.6 MiB/s wr, 833 op/s 2024-03-30T13:27:37.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:36 smithi187 ceph-mon[27499]: mgrmap e28: smithi187.jpkaud(active, since 93s), standbys: smithi159.fnasly 2024-03-30T13:27:37.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:36 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:37.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:36 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi159.fnasly", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:27:37.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:36 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi159.fnasly", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:27:37.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:36 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:27:37.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:36 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:27:37.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 129911 12.35 MB/sec execute 119 sec latency 83.734 ms 2024-03-30T13:27:38.059 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:37 smithi159 ceph-mon[24787]: Upgrade: Updating mgr.smithi159.fnasly 2024-03-30T13:27:38.059 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:37 smithi159 ceph-mon[24787]: Deploying daemon mgr.smithi159.fnasly on smithi159 2024-03-30T13:27:38.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:37 smithi187 ceph-mon[27499]: Upgrade: Updating mgr.smithi159.fnasly 2024-03-30T13:27:38.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:37 smithi187 ceph-mon[27499]: Deploying daemon mgr.smithi159.fnasly on smithi159 2024-03-30T13:27:38.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 130341 12.34 MB/sec execute 120 sec latency 55.398 ms 2024-03-30T13:27:39.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:38 smithi159 ceph-mon[24787]: pgmap v50: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.0 MiB/s wr, 504 op/s 2024-03-30T13:27:39.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:38 smithi187 ceph-mon[27499]: pgmap v50: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.0 MiB/s wr, 504 op/s 2024-03-30T13:27:39.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 130831 12.38 MB/sec execute 121 sec latency 29.001 ms 2024-03-30T13:27:40.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 131279 12.41 MB/sec execute 122 sec latency 31.719 ms 2024-03-30T13:27:41.109 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:40 smithi159 ceph-mon[24787]: pgmap v51: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.0 MiB/s wr, 504 op/s 2024-03-30T13:27:41.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:40 smithi187 ceph-mon[27499]: pgmap v51: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.0 MiB/s wr, 504 op/s 2024-03-30T13:27:41.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 131696 12.40 MB/sec execute 123 sec latency 39.266 ms 2024-03-30T13:27:42.889 INFO:teuthology.orchestra.run.smithi159.stdout: 5 131944 12.34 MB/sec execute 124 sec latency 43.991 ms 2024-03-30T13:27:43.063 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:42 smithi159 ceph-mon[24787]: pgmap v52: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 737 op/s 2024-03-30T13:27:43.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:42 smithi187 ceph-mon[27499]: pgmap v52: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 737 op/s 2024-03-30T13:27:43.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 132196 12.29 MB/sec execute 125 sec latency 54.048 ms 2024-03-30T13:27:44.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:44 smithi159 ceph-mon[24787]: pgmap v53: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-03-30T13:27:44.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 132604 12.26 MB/sec execute 126 sec latency 63.436 ms 2024-03-30T13:27:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:44 smithi187 ceph-mon[27499]: pgmap v53: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-03-30T13:27:45.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 132949 12.19 MB/sec execute 127 sec latency 57.636 ms 2024-03-30T13:27:46.012 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:45 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:46.013 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:45 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:46.013 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:45 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:27:46.013 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:45 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:46.013 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:45 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:45 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:45 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:45 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:27:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:45 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:45 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:46.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 133275 12.16 MB/sec execute 128 sec latency 48.298 ms 2024-03-30T13:27:47.019 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:46 smithi159 ceph-mon[24787]: pgmap v54: 97 pgs: 97 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 644 op/s 2024-03-30T13:27:47.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:46 smithi187 ceph-mon[27499]: pgmap v54: 97 pgs: 97 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 644 op/s 2024-03-30T13:27:47.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 133619 12.17 MB/sec execute 129 sec latency 63.002 ms 2024-03-30T13:27:48.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 134200 12.20 MB/sec execute 130 sec latency 35.246 ms 2024-03-30T13:27:49.074 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:48 smithi159 ceph-mon[24787]: pgmap v55: 97 pgs: 97 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-03-30T13:27:49.155 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:48 smithi187 ceph-mon[27499]: pgmap v55: 97 pgs: 97 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-03-30T13:27:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 134633 12.24 MB/sec execute 131 sec latency 27.803 ms 2024-03-30T13:27:50.115 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:49 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:27:50.116 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:49 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:50.116 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:49 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:50.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:49 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:27:50.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:49 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:50.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:49 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:50.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 135009 12.22 MB/sec execute 132 sec latency 41.504 ms 2024-03-30T13:27:51.137 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:50 smithi159 ceph-mon[24787]: pgmap v56: 97 pgs: 97 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-03-30T13:27:51.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:50 smithi187 ceph-mon[27499]: pgmap v56: 97 pgs: 97 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-03-30T13:27:51.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 135305 12.20 MB/sec execute 133 sec latency 63.798 ms 2024-03-30T13:27:52.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 135511 12.14 MB/sec execute 134 sec latency 56.962 ms 2024-03-30T13:27:52.917 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:52 smithi159 ceph-mon[24787]: pgmap v57: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.0 MiB/s wr, 694 op/s 2024-03-30T13:27:53.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:52 smithi187 ceph-mon[27499]: pgmap v57: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.0 MiB/s wr, 694 op/s 2024-03-30T13:27:53.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 135741 12.09 MB/sec execute 135 sec latency 44.173 ms 2024-03-30T13:27:54.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 136175 12.06 MB/sec execute 136 sec latency 44.044 ms 2024-03-30T13:27:55.208 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: pgmap v58: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-03-30T13:27:55.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:55.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:55.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: Standby manager daemon smithi159.fnasly restarted 2024-03-30T13:27:55.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: Standby manager daemon smithi159.fnasly started 2024-03-30T13:27:55.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi159.fnasly/crt"}]: dispatch 2024-03-30T13:27:55.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:27:55.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi159.fnasly/key"}]: dispatch 2024-03-30T13:27:55.209 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:54 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: pgmap v58: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: Standby manager daemon smithi159.fnasly restarted 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: Standby manager daemon smithi159.fnasly started 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi159.fnasly/crt"}]: dispatch 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi159.fnasly/key"}]: dispatch 2024-03-30T13:27:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:54 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:27:55.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 136641 12.02 MB/sec execute 137 sec latency 35.875 ms 2024-03-30T13:27:56.215 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:55 smithi159 ceph-mon[24787]: mgrmap e29: smithi187.jpkaud(active, since 112s), standbys: smithi159.fnasly 2024-03-30T13:27:56.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:55 smithi187 ceph-mon[27499]: mgrmap e29: smithi187.jpkaud(active, since 112s), standbys: smithi159.fnasly 2024-03-30T13:27:56.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 137035 12.04 MB/sec execute 138 sec latency 50.476 ms 2024-03-30T13:27:57.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:57 smithi159 ceph-mon[24787]: pgmap v59: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 656 op/s 2024-03-30T13:27:57.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:57 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:57.234 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:57 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:57 smithi187 ceph-mon[27499]: pgmap v59: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 656 op/s 2024-03-30T13:27:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:57 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:57 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:27:57.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 137582 12.05 MB/sec execute 139 sec latency 33.751 ms 2024-03-30T13:27:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 137943 12.07 MB/sec execute 140 sec latency 51.919 ms 2024-03-30T13:27:59.284 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:27:59 smithi159 ceph-mon[24787]: pgmap v60: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 437 op/s 2024-03-30T13:27:59.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:27:59 smithi187 ceph-mon[27499]: pgmap v60: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 437 op/s 2024-03-30T13:27:59.558 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:27:59.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 138255 12.08 MB/sec execute 141 sec latency 41.981 ms 2024-03-30T13:28:00.635 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:28:00.635 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (84s) 6s ago 10m 15.7M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:28:00.635 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (10m) 6s ago 10m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:28:00.635 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (10m) 11s ago 10m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:28:00.635 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 unknown 6s ago 10m - - 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (4m) 6s ago 4m 37.2M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (4m) 11s ago 4m 15.7M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (16s) 6s ago 11m 95.2M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (2m) 11s ago 9m 507M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 11c6be81ba54 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (12m) 6s ago 12m 55.0M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (9m) 11s ago 9m 43.4M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (4m) 6s ago 4m 255M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (4m) 11s ago 4m 84.0M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (58s) 6s ago 10m 11.5M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (27s) 11s ago 9m 9.98M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (9m) 6s ago 9m 474M 1936M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (8m) 6s ago 8m 410M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (8m) 6s ago 8m 324M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (7m) 6s ago 7m 440M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:28:00.636 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (7m) 11s ago 7m 453M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:28:00.637 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (7m) 11s ago 7m 514M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:28:00.637 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (6m) 11s ago 6m 421M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:28:00.637 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (6m) 11s ago 6m 545M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:28:00.637 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (29s) 6s ago 9m 49.2M - 2.33.4 514e6a882f6e 0eb5370a2601 2024-03-30T13:28:00.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 138582 12.04 MB/sec execute 142 sec latency 34.122 ms 2024-03-30T13:28:01.262 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:28:01.262 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-30T13:28:01.263 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:28:01.264 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:28:01.264 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:28:01.296 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:01 smithi159 ceph-mon[24787]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:01.296 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:01 smithi159 ceph-mon[24787]: pgmap v61: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 437 op/s 2024-03-30T13:28:01.296 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:01 smithi159 ceph-mon[24787]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:01.426 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:01 smithi187 ceph-mon[27499]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:01.426 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:01 smithi187 ceph-mon[27499]: pgmap v61: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 437 op/s 2024-03-30T13:28:01.426 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:01 smithi187 ceph-mon[27499]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:01.845 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr" 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "2/23 daemons upgraded", 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading mgr daemons", 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:28:01.846 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:28:01.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 138910 12.02 MB/sec execute 143 sec latency 56.397 ms 2024-03-30T13:28:02.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='client.24575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:02.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/64649187' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:28:02.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:28:02.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:28:02.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:02.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:28:02.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:28:02.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:28:02.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:02.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr fail", "who": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:02.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr fail", "who": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:02.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:02 smithi187 ceph-mon[27499]: osdmap e72: 8 total, 8 up, 8 in 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='client.24575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/64649187' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 172.21.15.187:0/3251473719' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr fail", "who": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:02.331 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "mgr fail", "who": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:02.331 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:02 smithi159 ceph-mon[24787]: osdmap e72: 8 total, 8 up, 8 in 2024-03-30T13:28:02.455 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:28:02.455 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:28:02.455 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:28:02.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 139179 11.97 MB/sec execute 144 sec latency 27.942 ms 2024-03-30T13:28:03.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: pgmap v62: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.1 MiB/s wr, 686 op/s 2024-03-30T13:28:03.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='client.24583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: pgmap v63: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: Upgrade: Need to upgrade myself (mgr.smithi187.jpkaud) 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: Upgrade: Need to upgrade myself (mgr.smithi187.jpkaud) 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: Failing over to other MGR 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1419820308' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd='[{"prefix": "mgr fail", "who": "smithi187.jpkaud"}]': finished 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: mgrmap e30: smithi159.fnasly(active, starting, since 0.863344s) 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:28:03.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:03.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:03 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:28:03.334 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: pgmap v62: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.1 MiB/s wr, 686 op/s 2024-03-30T13:28:03.334 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='client.24583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:03.334 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: pgmap v63: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: Upgrade: Need to upgrade myself (mgr.smithi187.jpkaud) 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: Upgrade: Need to upgrade myself (mgr.smithi187.jpkaud) 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: Failing over to other MGR 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/1419820308' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.24471 ' entity='mgr.smithi187.jpkaud' cmd='[{"prefix": "mgr fail", "who": "smithi187.jpkaud"}]': finished 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: mgrmap e30: smithi159.fnasly(active, starting, since 0.863344s) 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:28:03.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:03.336 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:03 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:28:03.890 INFO:teuthology.orchestra.run.smithi159.stdout: 5 139674 11.97 MB/sec execute 145 sec latency 34.430 ms 2024-03-30T13:28:04.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:04 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:28:04.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:04 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:28:04.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:04 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:28:04.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:04 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:28:04.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:04 smithi159 ceph-mon[24787]: mgrmap e31: smithi159.fnasly(active, since 1.86745s) 2024-03-30T13:28:04.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:04 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:28:04.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:04 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:28:04.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:04 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:28:04.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:04 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:28:04.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:04 smithi187 ceph-mon[27499]: mgrmap e31: smithi159.fnasly(active, since 1.86745s) 2024-03-30T13:28:04.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 140270 11.94 MB/sec execute 146 sec latency 19.287 ms 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: [30/Mar/2024:13:28:03] ENGINE Bus STARTING 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: [30/Mar/2024:13:28:03] ENGINE Serving on http://172.21.15.159:8765 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: [30/Mar/2024:13:28:03] ENGINE Serving on https://172.21.15.159:7150 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: [30/Mar/2024:13:28:03] ENGINE Bus STARTED 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: pgmap v3: 97 pgs: 97 active+clean; 321 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-30T13:28:05.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:05 smithi159 ceph-mon[24787]: Cluster is now healthy 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: [30/Mar/2024:13:28:03] ENGINE Bus STARTING 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: [30/Mar/2024:13:28:03] ENGINE Serving on http://172.21.15.159:8765 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: [30/Mar/2024:13:28:03] ENGINE Serving on https://172.21.15.159:7150 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: [30/Mar/2024:13:28:03] ENGINE Bus STARTED 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: pgmap v3: 97 pgs: 97 active+clean; 321 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-30T13:28:05.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:05 smithi187 ceph-mon[27499]: Cluster is now healthy 2024-03-30T13:28:05.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 141083 12.04 MB/sec execute 147 sec latency 24.429 ms 2024-03-30T13:28:06.348 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:06 smithi159 ceph-mon[24787]: pgmap v4: 97 pgs: 97 active+clean; 321 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:28:06.349 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:06 smithi159 ceph-mon[24787]: mgrmap e32: smithi159.fnasly(active, since 3s) 2024-03-30T13:28:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:06 smithi187 ceph-mon[27499]: pgmap v4: 97 pgs: 97 active+clean; 321 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:28:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:06 smithi187 ceph-mon[27499]: mgrmap e32: smithi159.fnasly(active, since 3s) 2024-03-30T13:28:06.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 141616 12.09 MB/sec execute 148 sec latency 40.437 ms 2024-03-30T13:28:07.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 142070 12.10 MB/sec execute 149 sec latency 28.656 ms 2024-03-30T13:28:08.368 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:08 smithi159 ceph-mon[24787]: pgmap v5: 97 pgs: 97 active+clean; 321 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:28:08.368 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:08 smithi159 ceph-mon[24787]: mgrmap e33: smithi159.fnasly(active, since 5s) 2024-03-30T13:28:08.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:08 smithi187 ceph-mon[27499]: pgmap v5: 97 pgs: 97 active+clean; 321 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:28:08.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:08 smithi187 ceph-mon[27499]: mgrmap e33: smithi159.fnasly(active, since 5s) 2024-03-30T13:28:08.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 142569 12.11 MB/sec execute 150 sec latency 41.525 ms 2024-03-30T13:28:09.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 142989 12.10 MB/sec execute 151 sec latency 57.040 ms 2024-03-30T13:28:10.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:10 smithi159 ceph-mon[24787]: pgmap v6: 97 pgs: 97 active+clean; 323 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 63 op/s 2024-03-30T13:28:10.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:10 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:10.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:10 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:10 smithi187 ceph-mon[27499]: pgmap v6: 97 pgs: 97 active+clean; 323 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 63 op/s 2024-03-30T13:28:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:10 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:10 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 143628 12.08 MB/sec execute 152 sec latency 32.051 ms 2024-03-30T13:28:11.658 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:11 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:11.658 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:11 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:11.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:11 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:11.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:11 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:11.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 144300 12.15 MB/sec execute 153 sec latency 22.208 ms 2024-03-30T13:28:12.668 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:12 smithi159 ceph-mon[24787]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 36 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-03-30T13:28:12.669 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:12 smithi159 ceph-mon[24787]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:28:12.669 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:12 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3117441029' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:28:12.669 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:12 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3117441029' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:28:12.669 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:12 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3117441029' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:28:12.669 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:12 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/3117441029' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:28:12.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 145033 12.21 MB/sec execute 154 sec latency 47.172 ms 2024-03-30T13:28:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:12 smithi187 ceph-mon[27499]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 36 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-03-30T13:28:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:12 smithi187 ceph-mon[27499]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:28:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:12 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3117441029' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:28:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:12 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3117441029' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:28:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:12 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3117441029' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:28:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:12 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/3117441029' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:28:13.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:13 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi187.jpkaud", "id": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:13.677 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:13 smithi159 ceph-mon[24787]: mgrmap e34: smithi159.fnasly(active, since 10s), standbys: smithi187.jpkaud 2024-03-30T13:28:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:13 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi187.jpkaud", "id": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:13 smithi187 ceph-mon[27499]: mgrmap e34: smithi159.fnasly(active, since 10s), standbys: smithi187.jpkaud 2024-03-30T13:28:13.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 145505 12.24 MB/sec execute 155 sec latency 37.656 ms 2024-03-30T13:28:14.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:14 smithi159 ceph-mon[24787]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 5.1 MiB/s wr, 471 op/s 2024-03-30T13:28:14.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 145982 12.24 MB/sec execute 156 sec latency 31.305 ms 2024-03-30T13:28:15.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:14 smithi187 ceph-mon[27499]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 5.1 MiB/s wr, 471 op/s 2024-03-30T13:28:15.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 146436 12.24 MB/sec execute 157 sec latency 46.023 ms 2024-03-30T13:28:16.410 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:16 smithi159 ceph-mon[24787]: pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 8.0 MiB/s wr, 762 op/s 2024-03-30T13:28:16.411 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:16 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:16.411 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:16 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:16.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:16 smithi187 ceph-mon[27499]: pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 8.0 MiB/s wr, 762 op/s 2024-03-30T13:28:16.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:16 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:16.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:16 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:16.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 146991 12.20 MB/sec execute 158 sec latency 37.010 ms 2024-03-30T13:28:17.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 147407 12.19 MB/sec execute 159 sec latency 35.894 ms 2024-03-30T13:28:18.182 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:18 smithi159 ceph-mon[24787]: pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 7.3 MiB/s wr, 697 op/s 2024-03-30T13:28:18.182 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:18 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:18.183 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:18 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:18 smithi187 ceph-mon[27499]: pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 7.3 MiB/s wr, 697 op/s 2024-03-30T13:28:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:18 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:18 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:18.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 147794 12.20 MB/sec execute 160 sec latency 49.664 ms 2024-03-30T13:28:19.219 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:19 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:19.220 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:19 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:28:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:19 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:19 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:28:19.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 148235 12.20 MB/sec execute 161 sec latency 49.750 ms 2024-03-30T13:28:20.733 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:20 smithi159 ceph-mon[24787]: pgmap v11: 97 pgs: 97 active+clean; 325 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 7.6 MiB/s wr, 714 op/s 2024-03-30T13:28:20.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 148646 12.21 MB/sec execute 162 sec latency 32.827 ms 2024-03-30T13:28:21.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:20 smithi187 ceph-mon[27499]: pgmap v11: 97 pgs: 97 active+clean; 325 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 7.6 MiB/s wr, 714 op/s 2024-03-30T13:28:21.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 149199 12.24 MB/sec execute 163 sec latency 26.742 ms 2024-03-30T13:28:22.743 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:22 smithi159 ceph-mon[24787]: pgmap v12: 97 pgs: 97 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 9.2 MiB/s wr, 958 op/s 2024-03-30T13:28:22.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 149684 12.25 MB/sec execute 164 sec latency 27.477 ms 2024-03-30T13:28:23.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:22 smithi187 ceph-mon[27499]: pgmap v12: 97 pgs: 97 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 9.2 MiB/s wr, 958 op/s 2024-03-30T13:28:23.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 150029 12.23 MB/sec execute 165 sec latency 58.566 ms 2024-03-30T13:28:24.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: pgmap v13: 97 pgs: 97 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 631 op/s 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:28:24.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:23 smithi159 ceph-mon[24787]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:28:24.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:24.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:24.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:24.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:28:24.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:28:24.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-30T13:28:24.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: pgmap v13: 97 pgs: 97 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 631 op/s 2024-03-30T13:28:24.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:28:24.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:28:24.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:23 smithi187 ceph-mon[27499]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:28:24.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 150448 12.19 MB/sec execute 166 sec latency 54.654 ms 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 934 op/s 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.023 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:25 smithi159 ceph-mon[24787]: Reconfiguring prometheus.smithi159 (dependencies changed)... 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 934 op/s 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:25.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:25 smithi187 ceph-mon[27499]: Reconfiguring prometheus.smithi159 (dependencies changed)... 2024-03-30T13:28:25.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 150879 12.16 MB/sec execute 167 sec latency 58.983 ms 2024-03-30T13:28:26.026 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:26 smithi159 ceph-mon[24787]: Reconfiguring daemon prometheus.smithi159 on smithi159 2024-03-30T13:28:26.026 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:26 smithi159 ceph-mon[24787]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-30T13:28:26.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:26 smithi187 ceph-mon[27499]: Reconfiguring daemon prometheus.smithi159 on smithi159 2024-03-30T13:28:26.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:26 smithi187 ceph-mon[27499]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-30T13:28:26.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 151381 12.19 MB/sec execute 168 sec latency 38.280 ms 2024-03-30T13:28:27.036 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:27 smithi159 ceph-mon[24787]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 574 op/s 2024-03-30T13:28:27.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:27 smithi187 ceph-mon[27499]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 574 op/s 2024-03-30T13:28:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: 5 151956 12.22 MB/sec execute 169 sec latency 32.599 ms 2024-03-30T13:28:28.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 152343 12.24 MB/sec execute 170 sec latency 62.196 ms 2024-03-30T13:28:29.814 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:29 smithi159 ceph-mon[24787]: pgmap v16: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 574 op/s 2024-03-30T13:28:29.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 152728 12.23 MB/sec execute 171 sec latency 39.939 ms 2024-03-30T13:28:30.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:29 smithi187 ceph-mon[27499]: pgmap v16: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 574 op/s 2024-03-30T13:28:30.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 153312 12.24 MB/sec execute 172 sec latency 36.993 ms 2024-03-30T13:28:31.582 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:31 smithi159 ceph-mon[24787]: pgmap v17: 97 pgs: 97 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.3 MiB/s wr, 847 op/s 2024-03-30T13:28:31.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 154030 12.25 MB/sec execute 173 sec latency 30.469 ms 2024-03-30T13:28:32.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:31 smithi187 ceph-mon[27499]: pgmap v17: 97 pgs: 97 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.3 MiB/s wr, 847 op/s 2024-03-30T13:28:32.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 154442 12.22 MB/sec execute 174 sec latency 68.209 ms 2024-03-30T13:28:33.059 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:28:33.622 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:33 smithi159 ceph-mon[24787]: pgmap v18: 97 pgs: 97 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-03-30T13:28:33.622 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:33 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:33.622 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:33 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:28:33.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 154834 12.24 MB/sec execute 175 sec latency 92.981 ms 2024-03-30T13:28:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:33 smithi187 ceph-mon[27499]: pgmap v18: 97 pgs: 97 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-03-30T13:28:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:33 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:33 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:28:34.266 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:28:34.266 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (117s) 18s ago 11m 17.8M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:28:34.266 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (11m) 18s ago 11m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:28:34.266 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (10m) 25s ago 10m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:28:34.266 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 unknown 18s ago 10m - - 2024-03-30T13:28:34.266 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (5m) 18s ago 5m 37.7M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:28:34.266 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (5m) 25s ago 5m 15.9M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:28:34.266 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (50s) 18s ago 12m 479M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283 running (2m) 25s ago 10m 112M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 11c6be81ba54 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (12m) 18s ago 12m 54.3M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (10m) 25s ago 10m 43.3M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (5m) 18s ago 5m 267M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (5m) 25s ago 5m 84.2M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (91s) 18s ago 10m 12.7M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (60s) 25s ago 10m 8144k - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (9m) 18s ago 9m 491M 1936M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (9m) 18s ago 9m 428M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (8m) 18s ago 8m 333M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (8m) 18s ago 8m 459M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (8m) 25s ago 8m 473M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (7m) 25s ago 7m 545M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (7m) 25s ago 7m 453M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:28:34.267 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (6m) 25s ago 6m 571M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:28:34.268 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 starting - - - - 2024-03-30T13:28:34.644 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:34 smithi159 ceph-mon[24787]: from='client.24613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:34.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:34 smithi159 ceph-mon[24787]: from='client.24617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:34.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:34 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:34.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:34 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:34.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:34 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:28:34.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:34 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:34.824 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:34 smithi187 ceph-mon[27499]: from='client.24613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:34.825 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:34 smithi187 ceph-mon[27499]: from='client.24617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:34.825 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:34 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:34.825 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:34 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:34.825 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:34 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:28:34.825 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:34 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:34.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 155302 12.23 MB/sec execute 176 sec latency 54.727 ms 2024-03-30T13:28:34.944 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:28:34.944 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:28:34.944 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:28:34.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:28:34.944 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:28:34.945 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:28:35.450 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:28:35.450 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:28:35.450 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:28:35.450 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:28:35.450 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:28:35.450 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr" 2024-03-30T13:28:35.451 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:28:35.451 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "2/23 daemons upgraded", 2024-03-30T13:28:35.451 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading mgr daemons", 2024-03-30T13:28:35.451 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:28:35.451 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:28:35.696 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:35 smithi159 ceph-mon[24787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:28:35.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:35 smithi159 ceph-mon[24787]: from='client.24619 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:35.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:35 smithi159 ceph-mon[24787]: pgmap v19: 97 pgs: 97 active+clean; 319 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 884 op/s 2024-03-30T13:28:35.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:35 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/555059064' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:28:35.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 155661 12.24 MB/sec execute 177 sec latency 58.051 ms 2024-03-30T13:28:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:35 smithi187 ceph-mon[27499]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:28:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:35 smithi187 ceph-mon[27499]: from='client.24619 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:35 smithi187 ceph-mon[27499]: pgmap v19: 97 pgs: 97 active+clean; 319 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 884 op/s 2024-03-30T13:28:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:35 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/555059064' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:28:36.092 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:28:36.092 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:28:36.092 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:28:36.704 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:36 smithi159 ceph-mon[24787]: from='client.24627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:36.705 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:36 smithi159 ceph-mon[24787]: from='client.? 172.21.15.159:0/3719443650' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:28:36.705 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:36 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:28:36.705 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:36 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:28:36.705 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:36 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:28:36.705 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:36 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:36.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:36 smithi187 ceph-mon[27499]: from='client.24627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:28:36.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:36 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3719443650' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:28:36.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:36 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:28:36.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:36 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:28:36.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:36 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:28:36.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:36 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:36.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 156044 12.26 MB/sec execute 178 sec latency 38.923 ms 2024-03-30T13:28:37.720 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:37 smithi159 ceph-mon[24787]: pgmap v20: 97 pgs: 97 active+clean; 319 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 584 op/s 2024-03-30T13:28:37.720 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:37 smithi159 ceph-mon[24787]: Upgrade: Updating mgr.smithi187.jpkaud 2024-03-30T13:28:37.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:37 smithi159 ceph-mon[24787]: Deploying daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:28:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:37 smithi187 ceph-mon[27499]: pgmap v20: 97 pgs: 97 active+clean; 319 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 584 op/s 2024-03-30T13:28:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:37 smithi187 ceph-mon[27499]: Upgrade: Updating mgr.smithi187.jpkaud 2024-03-30T13:28:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:37 smithi187 ceph-mon[27499]: Deploying daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:28:37.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 156477 12.24 MB/sec execute 179 sec latency 39.386 ms 2024-03-30T13:28:38.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 156850 12.23 MB/sec execute 180 sec latency 43.223 ms 2024-03-30T13:28:39.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:39 smithi159 ceph-mon[24787]: pgmap v21: 97 pgs: 97 active+clean; 319 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 584 op/s 2024-03-30T13:28:39.759 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:39 smithi187 ceph-mon[27499]: pgmap v21: 97 pgs: 97 active+clean; 319 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 584 op/s 2024-03-30T13:28:39.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 157281 12.21 MB/sec execute 181 sec latency 43.650 ms 2024-03-30T13:28:40.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 157647 12.17 MB/sec execute 182 sec latency 70.079 ms 2024-03-30T13:28:41.742 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:41 smithi159 ceph-mon[24787]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 829 op/s 2024-03-30T13:28:41.742 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:41 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:41.742 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:41 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:41.742 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:41 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:41.839 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:41 smithi187 ceph-mon[27499]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 829 op/s 2024-03-30T13:28:41.839 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:41 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:41.839 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:41 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:41.839 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:41 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:41.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 157903 12.13 MB/sec execute 183 sec latency 56.454 ms 2024-03-30T13:28:42.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 158245 12.14 MB/sec execute 184 sec latency 49.356 ms 2024-03-30T13:28:43.791 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:43 smithi159 ceph-mon[24787]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 551 op/s 2024-03-30T13:28:43.892 INFO:teuthology.orchestra.run.smithi159.stdout: 5 158663 12.14 MB/sec execute 185 sec latency 38.973 ms 2024-03-30T13:28:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:43 smithi187 ceph-mon[27499]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 551 op/s 2024-03-30T13:28:44.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 159189 12.16 MB/sec execute 186 sec latency 30.245 ms 2024-03-30T13:28:45.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:45 smithi187 ceph-mon[27499]: pgmap v24: 97 pgs: 97 active+clean; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 770 op/s 2024-03-30T13:28:45.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:45 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:45.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:45 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:45.830 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:45 smithi159 ceph-mon[24787]: pgmap v24: 97 pgs: 97 active+clean; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 770 op/s 2024-03-30T13:28:45.831 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:45 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:45.831 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:45 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:45.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 159582 12.18 MB/sec execute 187 sec latency 41.209 ms 2024-03-30T13:28:46.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 160197 12.19 MB/sec execute 188 sec latency 26.654 ms 2024-03-30T13:28:47.330 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:47 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:47.330 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:47 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:47.330 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:47 smithi187 ceph-mon[27499]: pgmap v25: 97 pgs: 97 active+clean; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-03-30T13:28:47.348 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:47 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:47.348 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:47 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:47.348 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:47 smithi159 ceph-mon[24787]: pgmap v25: 97 pgs: 97 active+clean; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-03-30T13:28:47.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 160686 12.20 MB/sec execute 189 sec latency 55.410 ms 2024-03-30T13:28:48.355 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:48 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:28:48.408 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:48 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:28:48.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 161145 12.16 MB/sec execute 190 sec latency 48.099 ms 2024-03-30T13:28:49.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:49 smithi159 ceph-mon[24787]: pgmap v26: 97 pgs: 97 active+clean; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-03-30T13:28:49.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:49 smithi159 ceph-mon[24787]: Standby manager daemon smithi187.jpkaud restarted 2024-03-30T13:28:49.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:49 smithi159 ceph-mon[24787]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:28:49.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:49 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/2241181092' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:28:49.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:49 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/2241181092' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:28:49.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:49 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/2241181092' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:28:49.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:49 smithi159 ceph-mon[24787]: from='mgr.? 172.21.15.187:0/2241181092' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:28:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 161567 12.14 MB/sec execute 191 sec latency 34.873 ms 2024-03-30T13:28:49.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:49 smithi187 ceph-mon[27499]: pgmap v26: 97 pgs: 97 active+clean; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-03-30T13:28:49.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:49 smithi187 ceph-mon[27499]: Standby manager daemon smithi187.jpkaud restarted 2024-03-30T13:28:49.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:49 smithi187 ceph-mon[27499]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:28:49.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:49 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/2241181092' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:28:49.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:49 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/2241181092' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:28:49.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:49 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/2241181092' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:28:49.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:49 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/2241181092' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:28:50.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 162035 12.17 MB/sec execute 192 sec latency 47.081 ms 2024-03-30T13:28:51.172 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:50 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:51.173 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:50 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:51.173 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:50 smithi159 ceph-mon[24787]: mgrmap e35: smithi159.fnasly(active, since 48s), standbys: smithi187.jpkaud 2024-03-30T13:28:51.173 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:50 smithi159 ceph-mon[24787]: pgmap v27: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 761 op/s 2024-03-30T13:28:51.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:50 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:51.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:50 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:51.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:50 smithi187 ceph-mon[27499]: mgrmap e35: smithi159.fnasly(active, since 48s), standbys: smithi187.jpkaud 2024-03-30T13:28:51.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:50 smithi187 ceph-mon[27499]: pgmap v27: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 761 op/s 2024-03-30T13:28:51.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 162539 12.17 MB/sec execute 193 sec latency 35.190 ms 2024-03-30T13:28:52.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 162847 12.18 MB/sec execute 194 sec latency 46.326 ms 2024-03-30T13:28:53.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: pgmap v28: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 516 op/s 2024-03-30T13:28:53.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:53.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:53.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:53.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:28:53.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: pgmap v29: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.7 MiB/s wr, 612 op/s 2024-03-30T13:28:53.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi159.fnasly"}]: dispatch 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi159.fnasly"}]: dispatch 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi159.fnasly"}]': finished 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi187.jpkaud"}]': finished 2024-03-30T13:28:53.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:53 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-30T13:28:53.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 163151 12.18 MB/sec execute 195 sec latency 50.118 ms 2024-03-30T13:28:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: pgmap v28: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 516 op/s 2024-03-30T13:28:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:28:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: pgmap v29: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.7 MiB/s wr, 612 op/s 2024-03-30T13:28:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi159.fnasly"}]: dispatch 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi159.fnasly"}]: dispatch 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi159.fnasly"}]': finished 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi187.jpkaud"}]: dispatch 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi187.jpkaud"}]': finished 2024-03-30T13:28:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:53 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-30T13:28:54.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:54 smithi159 ceph-mon[24787]: Upgrade: Setting container_image for all mgr 2024-03-30T13:28:54.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 163573 12.16 MB/sec execute 196 sec latency 33.235 ms 2024-03-30T13:28:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:54 smithi187 ceph-mon[27499]: Upgrade: Setting container_image for all mgr 2024-03-30T13:28:55.757 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:55 smithi159 ceph-mon[24787]: pgmap v30: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-03-30T13:28:55.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 163874 12.14 MB/sec execute 197 sec latency 61.144 ms 2024-03-30T13:28:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:55 smithi187 ceph-mon[27499]: pgmap v30: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-03-30T13:28:56.771 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:56 smithi159 ceph-mon[24787]: Upgrade: Updating mon.smithi159 2024-03-30T13:28:56.771 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:56 smithi159 ceph-mon[24787]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:56.771 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:56 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:28:56.771 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:56 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:28:56.771 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:56 smithi159 ceph-mon[24787]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:56.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 164310 12.13 MB/sec execute 198 sec latency 40.603 ms 2024-03-30T13:28:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:56 smithi187 ceph-mon[27499]: Upgrade: Updating mon.smithi159 2024-03-30T13:28:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:56 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:28:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:56 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:28:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:56 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:28:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:56 smithi187 ceph-mon[27499]: from='mgr.14780 172.21.15.159:0/1655121451' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:28:57.785 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:57 smithi159 ceph-mon[24787]: Deploying daemon mon.smithi159 on smithi159 2024-03-30T13:28:57.786 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:57 smithi159 ceph-mon[24787]: pgmap v31: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-03-30T13:28:57.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 164907 12.11 MB/sec execute 199 sec latency 42.258 ms 2024-03-30T13:28:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:57 smithi187 ceph-mon[27499]: Deploying daemon mon.smithi159 on smithi159 2024-03-30T13:28:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:28:57 smithi187 ceph-mon[27499]: pgmap v31: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-03-30T13:28:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 165558 12.16 MB/sec execute 200 sec latency 28.785 ms 2024-03-30T13:28:59.045 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:58 smithi159 systemd[1]: Stopping Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:28:59.251 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[24782]: 2024-03-30T13:28:59.196+0000 7f8d2bb70700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi159 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:28:59.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[24782]: 2024-03-30T13:28:59.196+0000 7f8d2bb70700 -1 mon.smithi159@0(leader) e2 *** Got Signal Terminated *** 2024-03-30T13:28:59.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:59 smithi159 podman[66795]: 2024-03-30 13:28:59.228667126 +0000 UTC m=+0.166924411 container died d58042490a7e0511e9cbe78be6c4f13b4bd32a4e5b2791c1600fa136ef0e6f3c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, ceph=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, RELEASE=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.buildah.version=1.19.8) 2024-03-30T13:28:59.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 166238 12.20 MB/sec execute 201 sec latency 34.480 ms 2024-03-30T13:29:00.049 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:59 smithi159 podman[66795]: 2024-03-30 13:28:59.992022444 +0000 UTC m=+0.930279726 container cleanup d58042490a7e0511e9cbe78be6c4f13b4bd32a4e5b2791c1600fa136ef0e6f3c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, architecture=x86_64, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, build-date=2022-05-03T08:36:31.336870, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, RELEASE=HEAD) 2024-03-30T13:29:00.050 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:28:59 smithi159 bash[66795]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159 2024-03-30T13:29:00.301 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:00 smithi159 podman[66809]: 2024-03-30 13:29:00.281077956 +0000 UTC m=+1.045896101 container remove d58042490a7e0511e9cbe78be6c4f13b4bd32a4e5b2791c1600fa136ef0e6f3c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-03-30T13:29:00.812 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:00 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159.service: Deactivated successfully. 2024-03-30T13:29:00.812 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:00 smithi159 systemd[1]: Stopped Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:29:00.812 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:00 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159.service: Consumed 17.213s CPU time. 2024-03-30T13:29:00.893 INFO:teuthology.orchestra.run.smithi159.stdout: 5 166698 12.23 MB/sec execute 202 sec latency 29.558 ms 2024-03-30T13:29:01.068 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 systemd[1]: Starting Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:29:01.318 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 podman[66977]: 2024-03-30 13:29:01.195388619 +0000 UTC m=+0.028853735 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:29:01.561 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 podman[66977]: 2024-03-30 13:29:01.500154394 +0000 UTC m=+0.333619485 container create ba7c32cec3016cd068a8a55c27b9a0b5afa45402281a06e6f30377e22501b23d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-30T13:29:01.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 167125 12.21 MB/sec execute 203 sec latency 48.664 ms 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 podman[66977]: 2024-03-30 13:29:01.827069626 +0000 UTC m=+0.660534738 container init ba7c32cec3016cd068a8a55c27b9a0b5afa45402281a06e6f30377e22501b23d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 podman[66977]: 2024-03-30 13:29:01.839848522 +0000 UTC m=+0.673313649 container start ba7c32cec3016cd068a8a55c27b9a0b5afa45402281a06e6f30377e22501b23d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: set uid:gid to 167:167 (ceph:ceph) 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev), process ceph-mon, pid 2 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: pidfile_write: ignore empty --pid-file 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: load: jerasure load: lrc 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: RocksDB version: 7.9.2 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Git sha 0 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Compile date 2024-03-29 15:06:16 2024-03-30T13:29:02.079 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: DB SUMMARY 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: DB Session ID: QI3P6IP66SL0L6R9T85P 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: CURRENT file: CURRENT 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: IDENTITY file: IDENTITY 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi159/store.db dir, Total Num: 1, files: 000030.sst 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi159/store.db: 000028.log size: 2913368 ; 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.error_if_exists: 0 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.create_if_missing: 0 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.paranoid_checks: 1 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.env: 0x563b804c4c60 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.fs: PosixFileSystem 2024-03-30T13:29:02.080 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.info_log: 0x563b81ce99a0 2024-03-30T13:29:02.081 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_file_opening_threads: 16 2024-03-30T13:29:02.082 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.statistics: (nil) 2024-03-30T13:29:02.082 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.use_fsync: 0 2024-03-30T13:29:02.082 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_log_file_size: 0 2024-03-30T13:29:02.082 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-30T13:29:02.082 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-30T13:29:02.082 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.keep_log_file_num: 1000 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.recycle_log_file_num: 0 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.allow_fallocate: 1 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.allow_mmap_reads: 0 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.allow_mmap_writes: 0 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.use_direct_reads: 0 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.create_missing_column_families: 0 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.db_log_dir: 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.wal_dir: 2024-03-30T13:29:02.083 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.advise_random_on_open: 1 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.db_write_buffer_size: 0 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.write_buffer_manager: 0x563b81cbc1e0 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.rate_limiter: (nil) 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-30T13:29:02.084 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.wal_recovery_mode: 2 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.enable_thread_tracking: 0 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.enable_pipelined_write: 0 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.unordered_write: 0 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.row_cache: None 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.wal_filter: None 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.allow_ingest_behind: 0 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.two_write_queues: 0 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.manual_wal_flush: 0 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.wal_compression: 0 2024-03-30T13:29:02.085 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.atomic_flush: 0 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.log_readahead_size: 0 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.best_efforts_recovery: 0 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.allow_data_in_errors: 0 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.db_host_id: __hostname__ 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_background_jobs: 2 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_background_compactions: -1 2024-03-30T13:29:02.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_subcompactions: 1 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_total_wal_size: 0 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_open_files: -1 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bytes_per_sync: 0 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-30T13:29:02.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_readahead_size: 0 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_background_flushes: -1 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Compression algorithms supported: 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: kZSTD supported: 0 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: kXpressCompression supported: 0 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: kLZ4HCCompression supported: 1 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: kZlibCompression supported: 1 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: kSnappyCompression supported: 1 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: kLZ4Compression supported: 1 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: kBZip2Compression supported: 0 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-30T13:29:02.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi159/store.db/MANIFEST-000015 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.merge_operator: 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_filter: None 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_filter_factory: None 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.sst_partitioner_factory: None 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-30T13:29:02.089 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563b81ce9aa0) 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: cache_index_and_filter_blocks: 1 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: pin_top_level_index_and_filter: 1 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: index_type: 0 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: data_block_index_type: 0 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: index_shortening: 1 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: checksum: 4 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: no_block_cache: 0 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_cache: 0x563b81ced090 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_cache_name: BinnedLRUCache 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_cache_options: 2024-03-30T13:29:02.090 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: capacity : 536870912 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: num_shard_bits : 4 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: strict_capacity_limit : 0 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: high_pri_pool_ratio: 0.000 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_cache_compressed: (nil) 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: persistent_cache: (nil) 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_size: 4096 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_size_deviation: 10 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_restart_interval: 16 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: index_block_restart_interval: 1 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: metadata_block_size: 4096 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: partition_filters: 0 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: use_delta_encoding: 1 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: filter_policy: bloomfilter 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: whole_key_filtering: 1 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: verify_compression: 0 2024-03-30T13:29:02.091 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: read_amp_bytes_per_bit: 0 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: format_version: 5 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: enable_index_compression: 1 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: block_align: 0 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: max_auto_readahead_size: 262144 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: prepopulate_block_cache: 0 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: initial_auto_readahead_size: 8192 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: num_file_reads_for_auto_readahead: 2 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.write_buffer_size: 33554432 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_write_buffer_number: 2 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression: NoCompression 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression: Disabled 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.prefix_extractor: nullptr 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-30T13:29:02.092 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.num_levels: 7 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-30T13:29:02.093 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.level: 32767 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.strategy: 0 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.enabled: false 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-30T13:29:02.094 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.target_file_size_base: 67108864 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-30T13:29:02.095 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.arena_block_size: 1048576 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.disable_auto_compactions: 0 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-30T13:29:02.096 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.table_properties_collectors: 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.inplace_update_support: 0 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.bloom_locality: 0 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.max_successive_merges: 0 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-30T13:29:02.097 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.paranoid_file_checks: 0 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.force_consistency_checks: 1 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.report_bg_io_stats: 0 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.ttl: 2592000 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.enable_blob_files: false 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.min_blob_size: 0 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.blob_file_size: 268435456 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-30T13:29:02.098 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.blob_file_starting_level: 0 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi159/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11795, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5ab294a9-fc7b-43c2-9a8b-95dbeeceea3d 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711805341876311, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711805341889287, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2652990, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11789, "largest_seqno": 13164, "table_properties": {"data_size": 2646401, "index_size": 4098, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13057, "raw_average_key_size": 22, "raw_value_size": 2633865, "raw_average_value_size": 4588, "num_data_blocks": 187, "num_entries": 574, "num_filter_entries": 574, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711805341, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5ab294a9-fc7b-43c2-9a8b-95dbeeceea3d", "db_session_id": "QI3P6IP66SL0L6R9T85P", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-03-30T13:29:02.099 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711805341889454, "job": 1, "event": "recovery_finished"} 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi159/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563b81dde000 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: DB pointer 0x563b81dc8000 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** DB Stats ** 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-30T13:29:02.100 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** Compaction Stats [default] ** 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: L0 1/0 2.53 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 351.2 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: L6 1/0 6.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Sum 2/0 8.87 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 351.2 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 351.2 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** Compaction Stats [default] ** 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 351.2 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-03-30T13:29:02.101 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Cumulative compaction: 0.00 GB write, 144.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Interval compaction: 0.00 GB write, 144.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Block cache BinnedLRUCache@0x563b81ced090#2 capacity: 512.00 MB usage: 6.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.102 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: starting mon.smithi159 rank 0 at public addrs [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] at bind addrs [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi159 fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: mon.smithi159@-1(???) e2 preinit fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: mon.smithi159@-1(???).mds e6 new map 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: mon.smithi159@-1(???).mds e6 print_map 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: e6 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: legacy client fscid: 1 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.103 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Filesystem 'foofs' (1) 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: fs_name foofs 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: epoch 4 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: created 2024-03-30T13:23:04.197862+0000 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: modified 2024-03-30T13:23:09.490705+0000 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: tableserver 0 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: root 0 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: session_timeout 60 2024-03-30T13:29:02.104 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: session_autoclose 300 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: max_file_size 1099511627776 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: max_xattr_size 65536 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: required_client_features {} 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: last_failure 0 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: last_failure_osd_epoch 0 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: max_mds 1 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: in 0 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: up {0=14476} 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: failed 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: damaged 2024-03-30T13:29:02.105 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: stopped 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: data_pools [3] 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: metadata_pool 2 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: inline_data disabled 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: balancer 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: bal_rank_mask -1 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: standby_count_wanted 1 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: qdb_cluster leader: 0 members: 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: [mds.foofs.smithi159.jwwwqm{0:14476} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.159:6834/3724310836,v1:172.21.15.159:6835/3724310836] compat {c=[1],r=[1],i=[7ff]}] 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: Standby daemons: 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout: [mds.foofs.smithi187.bkvdqd{-1:24305} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] compat {c=[1],r=[1],i=[7ff]}] 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: mon.smithi159@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-30T13:29:02.106 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: mon.smithi159@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-30T13:29:02.107 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: mon.smithi159@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-30T13:29:02.107 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: mon.smithi159@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-30T13:29:02.107 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:01 smithi159 ceph-mon[67002]: mon.smithi159@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-03-30T13:29:02.329 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 bash[66977]: ba7c32cec3016cd068a8a55c27b9a0b5afa45402281a06e6f30377e22501b23d 2024-03-30T13:29:02.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 systemd[1]: Started Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:29:02.583 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.7 MiB/s wr, 647 op/s 2024-03-30T13:29:02.583 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: mon.smithi159 calling monitor election 2024-03-30T13:29:02.583 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: mon.smithi159 is new leader, mons smithi159,smithi187 in quorum (ranks 0,1) 2024-03-30T13:29:02.583 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: monmap epoch 2 2024-03-30T13:29:02.583 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:29:02.583 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: last_changed 2024-03-30T13:18:05.820122+0000 2024-03-30T13:29:02.583 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: created 2024-03-30T13:15:51.155583+0000 2024-03-30T13:29:02.583 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: min_mon_release 17 (quincy) 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: election_strategy: 1 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: 0: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.smithi159 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.smithi187 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: osdmap e72: 8 total, 8 up, 8 in 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: mgrmap e35: smithi159.fnasly(active, since 60s), standbys: smithi187.jpkaud 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: mgrmap e36: smithi159.fnasly(active, since 60s), standbys: smithi187.jpkaud 2024-03-30T13:29:02.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:02 smithi159 ceph-mon[67002]: from='mgr.14780 ' entity='' 2024-03-30T13:29:02.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 167477 12.20 MB/sec execute 204 sec latency 65.160 ms 2024-03-30T13:29:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.7 MiB/s wr, 647 op/s 2024-03-30T13:29:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: mon.smithi159 calling monitor election 2024-03-30T13:29:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: mon.smithi159 is new leader, mons smithi159,smithi187 in quorum (ranks 0,1) 2024-03-30T13:29:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: monmap epoch 2 2024-03-30T13:29:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: last_changed 2024-03-30T13:18:05.820122+0000 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: created 2024-03-30T13:15:51.155583+0000 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: min_mon_release 17 (quincy) 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: election_strategy: 1 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: 0: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.smithi159 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.smithi187 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: osdmap e72: 8 total, 8 up, 8 in 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: mgrmap e35: smithi159.fnasly(active, since 60s), standbys: smithi187.jpkaud 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: mgrmap e36: smithi159.fnasly(active, since 60s), standbys: smithi187.jpkaud 2024-03-30T13:29:03.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:02 smithi187 ceph-mon[27499]: from='mgr.14780 ' entity='' 2024-03-30T13:29:03.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 167721 12.18 MB/sec execute 205 sec latency 69.478 ms 2024-03-30T13:29:04.615 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:04 smithi187 ceph-mon[27499]: mgrmap e37: smithi159.fnasly(active, since 61s), standbys: smithi187.jpkaud 2024-03-30T13:29:04.621 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:04 smithi159 ceph-mon[67002]: mgrmap e37: smithi159.fnasly(active, since 61s), standbys: smithi187.jpkaud 2024-03-30T13:29:04.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 167988 12.12 MB/sec execute 206 sec latency 74.607 ms 2024-03-30T13:29:05.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 168268 12.09 MB/sec execute 207 sec latency 59.629 ms 2024-03-30T13:29:06.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 168506 12.05 MB/sec execute 208 sec latency 38.972 ms 2024-03-30T13:29:07.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 168794 12.03 MB/sec execute 209 sec latency 61.399 ms 2024-03-30T13:29:08.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 169111 12.04 MB/sec execute 210 sec latency 76.267 ms 2024-03-30T13:29:09.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 169503 12.03 MB/sec execute 211 sec latency 63.127 ms 2024-03-30T13:29:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 169816 12.02 MB/sec execute 212 sec latency 46.238 ms 2024-03-30T13:29:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:10 smithi187 ceph-mon[27499]: Standby manager daemon smithi187.jpkaud restarted 2024-03-30T13:29:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:10 smithi187 ceph-mon[27499]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:29:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:10 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/2406483565' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:29:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:10 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/2406483565' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:29:11.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:10 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/2406483565' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:29:11.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:10 smithi187 ceph-mon[27499]: from='mgr.? 172.21.15.187:0/2406483565' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:29:11.065 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:10 smithi159 ceph-mon[67002]: Standby manager daemon smithi187.jpkaud restarted 2024-03-30T13:29:11.066 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:10 smithi159 ceph-mon[67002]: Standby manager daemon smithi187.jpkaud started 2024-03-30T13:29:11.066 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:10 smithi159 ceph-mon[67002]: from='mgr.? 172.21.15.187:0/2406483565' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/crt"}]: dispatch 2024-03-30T13:29:11.066 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:10 smithi159 ceph-mon[67002]: from='mgr.? 172.21.15.187:0/2406483565' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-30T13:29:11.066 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:10 smithi159 ceph-mon[67002]: from='mgr.? 172.21.15.187:0/2406483565' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi187.jpkaud/key"}]: dispatch 2024-03-30T13:29:11.066 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:10 smithi159 ceph-mon[67002]: from='mgr.? 172.21.15.187:0/2406483565' entity='mgr.smithi187.jpkaud' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-30T13:29:11.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 170020 12.00 MB/sec execute 213 sec latency 60.909 ms 2024-03-30T13:29:12.088 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:11 smithi159 ceph-mon[67002]: mgrmap e38: smithi159.fnasly(active, since 68s), standbys: smithi187.jpkaud 2024-03-30T13:29:12.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:11 smithi187 ceph-mon[27499]: mgrmap e38: smithi159.fnasly(active, since 68s), standbys: smithi187.jpkaud 2024-03-30T13:29:12.635 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187[27492]: 2024-03-30T13:29:12.379+0000 7fbbe2ec3700 -1 mon.smithi187@1(peon).osd e73 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-30T13:29:12.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: Active manager daemon smithi159.fnasly restarted 2024-03-30T13:29:12.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: Activating manager daemon smithi159.fnasly 2024-03-30T13:29:12.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: osdmap e73: 8 total, 8 up, 8 in 2024-03-30T13:29:12.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: mgrmap e39: smithi159.fnasly(active, starting, since 0.0073589s), standbys: smithi187.jpkaud 2024-03-30T13:29:12.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: failed to encode map e73 with expected crc 2024-03-30T13:29:12.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:29:12.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:29:12.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi187.jpkaud", "id": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:29:12.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:29:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:29:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:29:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:29:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:29:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:29:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:29:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:29:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:12 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:29:12.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 170288 11.99 MB/sec execute 214 sec latency 54.154 ms 2024-03-30T13:29:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: Active manager daemon smithi159.fnasly restarted 2024-03-30T13:29:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: Activating manager daemon smithi159.fnasly 2024-03-30T13:29:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: osdmap e73: 8 total, 8 up, 8 in 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: mgrmap e39: smithi159.fnasly(active, starting, since 0.0073589s), standbys: smithi187.jpkaud 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: failed to encode map e73 with expected crc 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi159.fnasly", "id": "smithi159.fnasly"}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr metadata", "who": "smithi187.jpkaud", "id": "smithi187.jpkaud"}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:29:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: Manager daemon smithi159.fnasly is now available 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/mirror_snapshot_schedule"}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:29:13.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:12 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi159.fnasly/trash_purge_schedule"}]: dispatch 2024-03-30T13:29:13.414 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:29:13.887 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:13 smithi159 ceph-mon[67002]: failed to encode map e73 with expected crc 2024-03-30T13:29:13.888 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:13 smithi159 ceph-mon[67002]: failed to encode map e73 with expected crc 2024-03-30T13:29:13.888 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:13 smithi159 ceph-mon[67002]: mgrmap e40: smithi159.fnasly(active, since 1.0106s), standbys: smithi187.jpkaud 2024-03-30T13:29:13.888 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:13 smithi159 ceph-mon[67002]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:13.888 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:13 smithi159 ceph-mon[67002]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:29:13.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 170520 11.95 MB/sec execute 215 sec latency 71.505 ms 2024-03-30T13:29:14.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:13 smithi187 ceph-mon[27499]: failed to encode map e73 with expected crc 2024-03-30T13:29:14.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:13 smithi187 ceph-mon[27499]: failed to encode map e73 with expected crc 2024-03-30T13:29:14.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:13 smithi187 ceph-mon[27499]: mgrmap e40: smithi159.fnasly(active, since 1.0106s), standbys: smithi187.jpkaud 2024-03-30T13:29:14.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:13 smithi187 ceph-mon[27499]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:14.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:13 smithi187 ceph-mon[27499]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (2m) 21s ago 11m 18.3M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (11m) 21s ago 11m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (11m) 29s ago 11m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 unknown 21s ago 11m - - 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (6m) 21s ago 6m 38.2M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (6m) 29s ago 6m 16.0M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (90s) 21s ago 13m 492M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (34s) 29s ago 11m 66.3M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:29:14.326 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (13m) 21s ago 13m 54.0M 2048M 17.2.0 e1d6a67b021e d58042490a7e 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (11m) 29s ago 11m 45.5M 2048M 17.2.0 e1d6a67b021e 9f317cdcecce 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (5m) 21s ago 5m 267M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (5m) 29s ago 5m 85.7M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (2m) 21s ago 11m 12.7M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (100s) 29s ago 11m 19.5M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (10m) 21s ago 10m 561M 1936M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (10m) 21s ago 9m 484M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (9m) 21s ago 9m 388M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (9m) 21s ago 9m 490M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (8m) 29s ago 8m 504M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (8m) 29s ago 8m 338M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (7m) 29s ago 7m 511M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:29:14.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (7m) 29s ago 7m 367M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:29:14.328 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (41s) 21s ago 10m 48.9M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:29:14.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 170752 11.92 MB/sec execute 216 sec latency 61.056 ms 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:29:14.906 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 3 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:29:14.907 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:29:14.922 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: [30/Mar/2024:13:29:13] ENGINE Bus STARTING 2024-03-30T13:29:14.922 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: [30/Mar/2024:13:29:13] ENGINE Serving on http://172.21.15.159:8765 2024-03-30T13:29:14.922 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:14.922 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: [30/Mar/2024:13:29:13] ENGINE Serving on https://172.21.15.159:7150 2024-03-30T13:29:14.922 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: [30/Mar/2024:13:29:13] ENGINE Bus STARTED 2024-03-30T13:29:14.923 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: from='client.24673 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:14.923 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-30T13:29:14.923 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: Cluster is now healthy 2024-03-30T13:29:14.923 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:14 smithi159 ceph-mon[67002]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: [30/Mar/2024:13:29:13] ENGINE Bus STARTING 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: [30/Mar/2024:13:29:13] ENGINE Serving on http://172.21.15.159:8765 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: [30/Mar/2024:13:29:13] ENGINE Serving on https://172.21.15.159:7150 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: [30/Mar/2024:13:29:13] ENGINE Bus STARTED 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: from='client.24673 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: Cluster is now healthy 2024-03-30T13:29:15.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:14 smithi187 ceph-mon[27499]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:29:15.410 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:29:15.410 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:29:15.410 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:29:15.410 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:29:15.410 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:29:15.410 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr" 2024-03-30T13:29:15.411 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:29:15.411 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "2/23 daemons upgraded", 2024-03-30T13:29:15.411 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "", 2024-03-30T13:29:15.411 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:29:15.411 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:29:15.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 170976 11.90 MB/sec execute 217 sec latency 52.876 ms 2024-03-30T13:29:15.942 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:15 smithi159 ceph-mon[67002]: mgrmap e41: smithi159.fnasly(active, since 2s), standbys: smithi187.jpkaud 2024-03-30T13:29:15.942 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:15 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3365076835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:29:15.942 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:15 smithi159 ceph-mon[67002]: from='client.24685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:16.011 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:29:16.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:15 smithi187 ceph-mon[27499]: mgrmap e41: smithi159.fnasly(active, since 2s), standbys: smithi187.jpkaud 2024-03-30T13:29:16.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:15 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/3365076835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:29:16.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:15 smithi187 ceph-mon[27499]: from='client.24685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:16.894 INFO:teuthology.orchestra.run.smithi159.stdout: 5 171235 11.89 MB/sec execute 218 sec latency 45.933 ms 2024-03-30T13:29:16.976 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:16 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1404153229' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:29:16.976 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:16 smithi159 ceph-mon[67002]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:29:16.977 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:16 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:16.977 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:16 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:17.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:16 smithi187 ceph-mon[27499]: from='client.? 172.21.15.159:0/1404153229' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:29:17.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:16 smithi187 ceph-mon[27499]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-30T13:29:17.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:16 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:17.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:16 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:17.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 171592 11.85 MB/sec execute 219 sec latency 53.850 ms 2024-03-30T13:29:17.999 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:17 smithi159 ceph-mon[67002]: mgrmap e42: smithi159.fnasly(active, since 4s), standbys: smithi187.jpkaud 2024-03-30T13:29:18.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:17 smithi187 ceph-mon[27499]: mgrmap e42: smithi159.fnasly(active, since 4s), standbys: smithi187.jpkaud 2024-03-30T13:29:18.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 171832 11.81 MB/sec execute 220 sec latency 77.548 ms 2024-03-30T13:29:19.338 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:19 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:19.338 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:19 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:19.338 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:19 smithi159 ceph-mon[67002]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 320 KiB/s wr, 12 op/s 2024-03-30T13:29:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:19 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:19 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:19.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:19 smithi187 ceph-mon[27499]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 320 KiB/s wr, 12 op/s 2024-03-30T13:29:19.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 172058 11.77 MB/sec execute 221 sec latency 63.721 ms 2024-03-30T13:29:20.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 172304 11.76 MB/sec execute 222 sec latency 58.217 ms 2024-03-30T13:29:21.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:21 smithi159 ceph-mon[67002]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 2.5 MiB/s wr, 307 op/s 2024-03-30T13:29:21.646 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:21 smithi187 ceph-mon[27499]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 2.5 MiB/s wr, 307 op/s 2024-03-30T13:29:21.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 172569 11.76 MB/sec execute 223 sec latency 57.322 ms 2024-03-30T13:29:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:22 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:22 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:22 smithi187 ceph-mon[27499]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 MiB/s wr, 238 op/s 2024-03-30T13:29:22.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 172865 11.74 MB/sec execute 224 sec latency 67.688 ms 2024-03-30T13:29:22.930 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:22 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:22.930 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:22 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:22.931 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:22 smithi159 ceph-mon[67002]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 MiB/s wr, 238 op/s 2024-03-30T13:29:23.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 173258 11.74 MB/sec execute 225 sec latency 57.843 ms 2024-03-30T13:29:24.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 173572 11.73 MB/sec execute 226 sec latency 32.130 ms 2024-03-30T13:29:25.467 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:25 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:25.467 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:25 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:25.467 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:25 smithi159 ceph-mon[67002]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 3.3 MiB/s wr, 368 op/s 2024-03-30T13:29:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:25 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:25 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:25 smithi187 ceph-mon[27499]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 3.3 MiB/s wr, 368 op/s 2024-03-30T13:29:25.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 173947 11.73 MB/sec execute 227 sec latency 34.537 ms 2024-03-30T13:29:26.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 174258 11.71 MB/sec execute 228 sec latency 48.554 ms 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.0 MiB/s wr, 337 op/s 2024-03-30T13:29:26.993 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:26 smithi159 ceph-mon[67002]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.conf 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.0 MiB/s wr, 337 op/s 2024-03-30T13:29:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:26 smithi187 ceph-mon[27499]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:29:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 174521 11.69 MB/sec execute 229 sec latency 44.876 ms 2024-03-30T13:29:28.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:29:28.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:29:28.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:29:28.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: pgmap v11: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 3.4 MiB/s wr, 371 op/s 2024-03-30T13:29:28.271 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.271 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.271 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:29:28.271 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:29:28.271 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-30T13:29:28.271 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.271 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: Updating smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: Updating smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: pgmap v11: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 3.4 MiB/s wr, 371 op/s 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:29:28.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:29:28.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-30T13:29:28.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:28.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:28 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:29:28.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 174742 11.67 MB/sec execute 230 sec latency 55.988 ms 2024-03-30T13:29:29.285 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:29 smithi159 ceph-mon[67002]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-30T13:29:29.431 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:29 smithi187 ceph-mon[27499]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-30T13:29:29.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 175041 11.65 MB/sec execute 231 sec latency 77.518 ms 2024-03-30T13:29:30.558 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:30 smithi159 ceph-mon[67002]: pgmap v12: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 5.7 MiB/s wr, 590 op/s 2024-03-30T13:29:30.559 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:30 smithi159 ceph-mon[67002]: Upgrade: Updating mon.smithi187 2024-03-30T13:29:30.559 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:30 smithi159 ceph-mon[67002]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:30.559 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:29:30.559 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:29:30.559 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:30.559 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:30 smithi159 ceph-mon[67002]: Deploying daemon mon.smithi187 on smithi187 2024-03-30T13:29:30.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:30 smithi187 ceph-mon[27499]: pgmap v12: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 5.7 MiB/s wr, 590 op/s 2024-03-30T13:29:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:30 smithi187 ceph-mon[27499]: Upgrade: Updating mon.smithi187 2024-03-30T13:29:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:30 smithi187 ceph-mon[27499]: from='mgr.34104 ' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:30 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:29:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:30 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:29:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:30 smithi187 ceph-mon[27499]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:30 smithi187 ceph-mon[27499]: Deploying daemon mon.smithi187 on smithi187 2024-03-30T13:29:30.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 175404 11.61 MB/sec execute 232 sec latency 55.126 ms 2024-03-30T13:29:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:31 smithi187 systemd[1]: Stopping Ceph mon.smithi187 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:29:31.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 175720 11.60 MB/sec execute 233 sec latency 46.781 ms 2024-03-30T13:29:32.215 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187[27492]: 2024-03-30T13:29:31.814+0000 7fbbe86ce700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi187 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:29:32.215 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187[27492]: 2024-03-30T13:29:31.814+0000 7fbbe86ce700 -1 mon.smithi187@1(peon) e2 *** Got Signal Terminated *** 2024-03-30T13:29:32.215 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:31 smithi187 podman[53444]: 2024-03-30 13:29:31.967842396 +0000 UTC m=+0.281371113 container died 9f317cdceccec009300e17c0ad4860a3c36758b0f37a06ff5a2a883b003d0e3d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, io.openshift.tags=base centos centos-stream, release=754, ceph=True, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, distribution-scope=public, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.expose-services=, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.buildah.version=1.19.8, version=8) 2024-03-30T13:29:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:32 smithi187 podman[53444]: 2024-03-30 13:29:32.50576836 +0000 UTC m=+0.819297075 container cleanup 9f317cdceccec009300e17c0ad4860a3c36758b0f37a06ff5a2a883b003d0e3d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, vcs-type=git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , release=754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, version=8, io.k8s.display-name=CentOS Stream 8) 2024-03-30T13:29:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:32 smithi187 bash[53444]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187 2024-03-30T13:29:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:32 smithi187 podman[53458]: 2024-03-30 13:29:32.707876683 +0000 UTC m=+0.737856101 container remove 9f317cdceccec009300e17c0ad4860a3c36758b0f37a06ff5a2a883b003d0e3d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, ceph=True, release=754, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-type=git, distribution-scope=public) 2024-03-30T13:29:32.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 176061 11.60 MB/sec execute 234 sec latency 58.768 ms 2024-03-30T13:29:33.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:32 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi187.service: Deactivated successfully. 2024-03-30T13:29:33.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:32 smithi187 systemd[1]: Stopped Ceph mon.smithi187 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:29:33.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:32 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi187.service: Consumed 7.411s CPU time. 2024-03-30T13:29:33.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 systemd[1]: Starting Ceph mon.smithi187 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:29:33.541 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 podman[53586]: 2024-03-30 13:29:33.307021289 +0000 UTC m=+0.019524757 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:29:33.541 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 podman[53586]: 2024-03-30 13:29:33.426322593 +0000 UTC m=+0.138826060 container create 88b8025f24dd2ea3b4d42ad60f11bc72962f4344245fde97ce18bd23d8a8470a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True) 2024-03-30T13:29:33.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 176432 11.61 MB/sec execute 235 sec latency 54.838 ms 2024-03-30T13:29:33.913 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 podman[53586]: 2024-03-30 13:29:33.654411505 +0000 UTC m=+0.366914973 container init 88b8025f24dd2ea3b4d42ad60f11bc72962f4344245fde97ce18bd23d8a8470a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:29:33.913 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 podman[53586]: 2024-03-30 13:29:33.657970606 +0000 UTC m=+0.370474074 container start 88b8025f24dd2ea3b4d42ad60f11bc72962f4344245fde97ce18bd23d8a8470a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: set uid:gid to 167:167 (ceph:ceph) 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev), process ceph-mon, pid 2 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: pidfile_write: ignore empty --pid-file 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: load: jerasure load: lrc 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: RocksDB version: 7.9.2 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Git sha 0 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Compile date 2024-03-29 15:06:16 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: DB SUMMARY 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: DB Session ID: M0HZC9JJCX1XVK8B7W5V 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: CURRENT file: CURRENT 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: IDENTITY file: IDENTITY 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi187/store.db dir, Total Num: 1, files: 000024.sst 2024-03-30T13:29:33.914 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi187/store.db: 000022.log size: 7634474 ; 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.error_if_exists: 0 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.create_if_missing: 0 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.paranoid_checks: 1 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.env: 0x55ecce73ac60 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.fs: PosixFileSystem 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.info_log: 0x55ecd09259a0 2024-03-30T13:29:33.915 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_file_opening_threads: 16 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.statistics: (nil) 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.use_fsync: 0 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_log_file_size: 0 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.keep_log_file_num: 1000 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.recycle_log_file_num: 0 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.allow_fallocate: 1 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.allow_mmap_reads: 0 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.allow_mmap_writes: 0 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.use_direct_reads: 0 2024-03-30T13:29:33.916 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.create_missing_column_families: 0 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.db_log_dir: 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.wal_dir: 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.advise_random_on_open: 1 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.db_write_buffer_size: 0 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.write_buffer_manager: 0x55ecd08f81e0 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-30T13:29:33.917 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.rate_limiter: (nil) 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.wal_recovery_mode: 2 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.enable_thread_tracking: 0 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.enable_pipelined_write: 0 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.unordered_write: 0 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.row_cache: None 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.wal_filter: None 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.allow_ingest_behind: 0 2024-03-30T13:29:33.918 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.two_write_queues: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.manual_wal_flush: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.wal_compression: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.atomic_flush: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.log_readahead_size: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.best_efforts_recovery: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.allow_data_in_errors: 0 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.db_host_id: __hostname__ 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_background_jobs: 2 2024-03-30T13:29:33.919 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_background_compactions: -1 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_subcompactions: 1 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_total_wal_size: 0 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_open_files: -1 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bytes_per_sync: 0 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_readahead_size: 0 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_background_flushes: -1 2024-03-30T13:29:33.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Compression algorithms supported: 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: kZSTD supported: 0 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: kXpressCompression supported: 0 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: kLZ4HCCompression supported: 1 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: kZlibCompression supported: 1 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: kSnappyCompression supported: 1 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: kLZ4Compression supported: 1 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: kBZip2Compression supported: 0 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi187/store.db/MANIFEST-000009 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.merge_operator: 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_filter: None 2024-03-30T13:29:33.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_filter_factory: None 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.sst_partitioner_factory: None 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ecd0925aa0) 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: cache_index_and_filter_blocks: 1 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: pin_top_level_index_and_filter: 1 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: index_type: 0 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: data_block_index_type: 0 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: index_shortening: 1 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: checksum: 4 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: no_block_cache: 0 2024-03-30T13:29:33.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_cache: 0x55ecd0929090 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_cache_name: BinnedLRUCache 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_cache_options: 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: capacity : 536870912 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: num_shard_bits : 4 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: strict_capacity_limit : 0 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: high_pri_pool_ratio: 0.000 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_cache_compressed: (nil) 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: persistent_cache: (nil) 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_size: 4096 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_size_deviation: 10 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_restart_interval: 16 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: index_block_restart_interval: 1 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: metadata_block_size: 4096 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: partition_filters: 0 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: use_delta_encoding: 1 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: filter_policy: bloomfilter 2024-03-30T13:29:33.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: whole_key_filtering: 1 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: verify_compression: 0 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: read_amp_bytes_per_bit: 0 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: format_version: 5 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: enable_index_compression: 1 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: block_align: 0 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: max_auto_readahead_size: 262144 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: prepopulate_block_cache: 0 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: initial_auto_readahead_size: 8192 2024-03-30T13:29:33.924 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: num_file_reads_for_auto_readahead: 2 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.write_buffer_size: 33554432 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_write_buffer_number: 2 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression: NoCompression 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression: Disabled 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.prefix_extractor: nullptr 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.num_levels: 7 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-30T13:29:33.925 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.level: 32767 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.strategy: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-30T13:29:33.926 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.enabled: false 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.target_file_size_base: 67108864 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-30T13:29:33.927 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.arena_block_size: 1048576 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.disable_auto_compactions: 0 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-30T13:29:33.928 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.table_properties_collectors: 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.inplace_update_support: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.bloom_locality: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.max_successive_merges: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.paranoid_file_checks: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.force_consistency_checks: 1 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.report_bg_io_stats: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.ttl: 2592000 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-30T13:29:33.929 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.enable_blob_files: false 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.min_blob_size: 0 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.blob_file_size: 268435456 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.blob_file_starting_level: 0 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi187/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12194, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-03-30T13:29:33.930 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b0c0e1bd-c0df-4181-8470-ceba69edad35 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711805373689098, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711805373709460, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4621996, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12191, "largest_seqno": 14309, "table_properties": {"data_size": 4613043, "index_size": 5885, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2053, "raw_key_size": 18647, "raw_average_key_size": 22, "raw_value_size": 4595314, "raw_average_value_size": 5631, "num_data_blocks": 267, "num_entries": 816, "num_filter_entries": 816, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711805373, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b0c0e1bd-c0df-4181-8470-ceba69edad35", "db_session_id": "M0HZC9JJCX1XVK8B7W5V", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711805373709576, "job": 1, "event": "recovery_finished"} 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi187/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ecd0a1a000 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: DB pointer 0x55ecd0a04000 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** DB Stats ** 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-30T13:29:33.931 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** Compaction Stats [default] ** 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: L0 1/0 4.41 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 545.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: L6 1/0 6.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Sum 2/0 10.75 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 545.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 545.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** Compaction Stats [default] ** 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-30T13:29:33.932 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 545.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Cumulative compaction: 0.00 GB write, 134.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Interval compaction: 0.00 GB write, 134.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Block cache BinnedLRUCache@0x55ecd0929090#2 capacity: 512.00 MB usage: 8.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.933 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: starting mon.smithi187 rank 1 at public addrs [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] at bind addrs [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi187 fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: mon.smithi187@-1(???) e2 preinit fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: mon.smithi187@-1(???).mds e6 new map 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: mon.smithi187@-1(???).mds e6 print_map 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: e6 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: legacy client fscid: 1 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Filesystem 'foofs' (1) 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: fs_name foofs 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: epoch 4 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: created 2024-03-30T13:23:04.197862+0000 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: modified 2024-03-30T13:23:09.490705+0000 2024-03-30T13:29:33.934 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: tableserver 0 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: root 0 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: session_timeout 60 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: session_autoclose 300 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: max_file_size 1099511627776 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: max_xattr_size 65536 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: required_client_features {} 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: last_failure 0 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: last_failure_osd_epoch 0 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: max_mds 1 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: in 0 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: up {0=14476} 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: failed 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: damaged 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: stopped 2024-03-30T13:29:33.935 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: data_pools [3] 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: metadata_pool 2 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: inline_data disabled 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: balancer 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: bal_rank_mask -1 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: standby_count_wanted 1 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: qdb_cluster leader: 0 members: 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: [mds.foofs.smithi159.jwwwqm{0:14476} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.159:6834/3724310836,v1:172.21.15.159:6835/3724310836] compat {c=[1],r=[1],i=[7ff]}] 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: Standby daemons: 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout: [mds.foofs.smithi187.bkvdqd{-1:24305} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] compat {c=[1],r=[1],i=[7ff]}] 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: mon.smithi187@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: mon.smithi187@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: mon.smithi187@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: mon.smithi187@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-30T13:29:33.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: mon.smithi187@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-03-30T13:29:33.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 bash[53586]: 88b8025f24dd2ea3b4d42ad60f11bc72962f4344245fde97ce18bd23d8a8470a 2024-03-30T13:29:33.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 systemd[1]: Started Ceph mon.smithi187 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:29:33.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: pgmap v13: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 399 op/s 2024-03-30T13:29:33.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:29:33.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:33 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:29:34.145 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:33 smithi159 ceph-mon[67002]: pgmap v13: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 399 op/s 2024-03-30T13:29:34.145 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:33 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:29:34.145 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:33 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:29:34.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 177019 11.63 MB/sec execute 236 sec latency 28.412 ms 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: 0: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.smithi159 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.smithi187 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: osdmap e73: 8 total, 8 up, 8 in 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: mgrmap e42: smithi159.fnasly(active, since 21s), standbys: smithi187.jpkaud 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: mon.smithi159 calling monitor election 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: mon.smithi187 calling monitor election 2024-03-30T13:29:35.020 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: mon.smithi159 is new leader, mons smithi159,smithi187 in quorum (ranks 0,1) 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: monmap epoch 3 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: last_changed 2024-03-30T13:29:33.913061+0000 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: created 2024-03-30T13:15:51.155583+0000 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: min_mon_release 19 (squid) 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: election_strategy: 1 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: 0: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.smithi159 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.smithi187 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: osdmap e73: 8 total, 8 up, 8 in 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: mgrmap e42: smithi159.fnasly(active, since 21s), standbys: smithi187.jpkaud 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:29:35.021 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:29:35.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:35.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:35.022 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:29:35.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: 0: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.smithi159 2024-03-30T13:29:35.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.smithi187 2024-03-30T13:29:35.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:29:35.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: osdmap e73: 8 total, 8 up, 8 in 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: mgrmap e42: smithi159.fnasly(active, since 21s), standbys: smithi187.jpkaud 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi159"}]: dispatch 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mon metadata", "id": "smithi187"}]: dispatch 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: mon.smithi159 calling monitor election 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: mon.smithi187 calling monitor election 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: mon.smithi159 is new leader, mons smithi159,smithi187 in quorum (ranks 0,1) 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: monmap epoch 3 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: last_changed 2024-03-30T13:29:33.913061+0000 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: created 2024-03-30T13:15:51.155583+0000 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: min_mon_release 19 (squid) 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: election_strategy: 1 2024-03-30T13:29:35.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: 0: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.smithi159 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.smithi187 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: osdmap e73: 8 total, 8 up, 8 in 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: mgrmap e42: smithi159.fnasly(active, since 21s), standbys: smithi187.jpkaud 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:35.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:29:35.895 INFO:teuthology.orchestra.run.smithi159.stdout: 5 177509 11.64 MB/sec execute 237 sec latency 24.987 ms 2024-03-30T13:29:36.171 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:35 smithi159 ceph-mon[67002]: pgmap v15: 97 pgs: 97 active+clean; 330 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-03-30T13:29:36.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:35 smithi187 ceph-mon[53604]: pgmap v15: 97 pgs: 97 active+clean; 330 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-03-30T13:29:36.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 178028 11.64 MB/sec execute 238 sec latency 32.441 ms 2024-03-30T13:29:37.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 178453 11.65 MB/sec execute 239 sec latency 40.747 ms 2024-03-30T13:29:38.455 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:38 smithi159 ceph-mon[67002]: pgmap v16: 97 pgs: 97 active+clean; 330 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-03-30T13:29:38.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:38 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:38.456 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:38 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:38.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:38 smithi187 ceph-mon[53604]: pgmap v16: 97 pgs: 97 active+clean; 330 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-03-30T13:29:38.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:38 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:38.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:38 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:38.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 178940 11.62 MB/sec execute 240 sec latency 47.614 ms 2024-03-30T13:29:39.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 179388 11.63 MB/sec execute 241 sec latency 37.292 ms 2024-03-30T13:29:40.387 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:40 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:40.388 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:40 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:40.388 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:40 smithi159 ceph-mon[67002]: pgmap v17: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.1 MiB/s wr, 691 op/s 2024-03-30T13:29:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:40 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:40 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:40 smithi187 ceph-mon[53604]: pgmap v17: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.1 MiB/s wr, 691 op/s 2024-03-30T13:29:40.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 179897 11.66 MB/sec execute 242 sec latency 43.851 ms 2024-03-30T13:29:41.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 180494 11.67 MB/sec execute 243 sec latency 30.085 ms 2024-03-30T13:29:42.403 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:42 smithi159 ceph-mon[67002]: pgmap v18: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 487 op/s 2024-03-30T13:29:42.479 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:42 smithi187 ceph-mon[53604]: pgmap v18: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 487 op/s 2024-03-30T13:29:42.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 180944 11.69 MB/sec execute 244 sec latency 37.997 ms 2024-03-30T13:29:43.669 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:29:43.670 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:29:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:29:43.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:43.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 181316 11.67 MB/sec execute 245 sec latency 36.932 ms 2024-03-30T13:29:44.683 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:44 smithi159 ceph-mon[67002]: pgmap v19: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 505 op/s 2024-03-30T13:29:44.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:44 smithi159 ceph-mon[67002]: pgmap v20: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-03-30T13:29:44.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:44 smithi159 ceph-mon[67002]: Reconfiguring mon.smithi159 (monmap changed)... 2024-03-30T13:29:44.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:44 smithi159 ceph-mon[67002]: Reconfiguring daemon mon.smithi159 on smithi159 2024-03-30T13:29:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:44 smithi187 ceph-mon[53604]: pgmap v19: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 505 op/s 2024-03-30T13:29:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:44 smithi187 ceph-mon[53604]: pgmap v20: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-03-30T13:29:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:44 smithi187 ceph-mon[53604]: Reconfiguring mon.smithi159 (monmap changed)... 2024-03-30T13:29:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:44 smithi187 ceph-mon[53604]: Reconfiguring daemon mon.smithi159 on smithi159 2024-03-30T13:29:44.901 INFO:teuthology.orchestra.run.smithi159.stdout: 5 181725 11.68 MB/sec execute 246 sec latency 47.147 ms 2024-03-30T13:29:45.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 182116 11.66 MB/sec execute 247 sec latency 82.743 ms 2024-03-30T13:29:46.502 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:46 smithi187 ceph-mon[53604]: pgmap v21: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-03-30T13:29:46.502 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:46.502 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:46.502 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi159.fnasly", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:29:46.634 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:29:46.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:46 smithi159 ceph-mon[67002]: pgmap v21: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-03-30T13:29:46.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:46.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:46.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi159.fnasly", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:29:46.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:29:46.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:46.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:29:46.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:46.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 182495 11.64 MB/sec execute 248 sec latency 38.825 ms 2024-03-30T13:29:47.658 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:29:47.658 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (3m) 23s ago 12m 18.3M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:29:47.658 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (12m) 23s ago 12m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:29:47.658 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (11m) 9s ago 11m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:29:47.658 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 unknown 23s ago 12m - - 2024-03-30T13:29:47.658 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (6m) 23s ago 6m 37.9M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (6m) 9s ago 6m 16.6M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (2m) 23s ago 13m 476M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (67s) 9s ago 11m 448M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (46s) 23s ago 13m 30.6M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (14s) 9s ago 11m 24.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (6m) 23s ago 6m 275M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (6m) 9s ago 6m 86.0M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (2m) 23s ago 12m 12.7M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (2m) 9s ago 11m 19.9M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (11m) 23s ago 11m 326M 1936M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (10m) 23s ago 10m 515M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (10m) 23s ago 10m 429M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (9m) 23s ago 9m 520M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (9m) 9s ago 9m 539M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:29:47.659 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (8m) 9s ago 8m 407M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:29:47.660 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (8m) 9s ago 8m 567M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:29:47.660 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (8m) 9s ago 8m 434M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:29:47.660 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (74s) 23s ago 11m 55.5M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:29:47.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:47 smithi159 ceph-mon[67002]: Reconfiguring mgr.smithi159.fnasly (monmap changed)... 2024-03-30T13:29:47.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:47 smithi159 ceph-mon[67002]: Reconfiguring daemon mgr.smithi159.fnasly on smithi159 2024-03-30T13:29:47.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:47 smithi159 ceph-mon[67002]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:47.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:47 smithi187 ceph-mon[53604]: Reconfiguring mgr.smithi159.fnasly (monmap changed)... 2024-03-30T13:29:47.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:47 smithi187 ceph-mon[53604]: Reconfiguring daemon mgr.smithi159.fnasly on smithi159 2024-03-30T13:29:47.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:47 smithi187 ceph-mon[53604]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:47.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 182875 11.63 MB/sec execute 249 sec latency 48.582 ms 2024-03-30T13:29:48.273 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:29:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-30T13:29:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 4 2024-03-30T13:29:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:29:48.275 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:29:48.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:48 smithi159 ceph-mon[67002]: pgmap v22: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-03-30T13:29:48.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:48 smithi159 ceph-mon[67002]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:48.526 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:48 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1609149350' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:29:48.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:48 smithi187 ceph-mon[53604]: pgmap v22: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-03-30T13:29:48.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:48 smithi187 ceph-mon[53604]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:48.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:48 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1609149350' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:29:48.875 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:29:48.875 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:29:48.875 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr" 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "4/23 daemons upgraded", 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading mon daemons", 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:29:48.876 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:29:48.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 183489 11.67 MB/sec execute 250 sec latency 43.875 ms 2024-03-30T13:29:49.529 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:49 smithi159 ceph-mon[67002]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:49.530 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:49.530 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:49.530 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi159", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:29:49.530 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:49.583 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:29:49.583 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:29:49.583 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:29:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 184119 11.70 MB/sec execute 251 sec latency 30.351 ms 2024-03-30T13:29:50.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:49 smithi187 ceph-mon[53604]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:50.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:50.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:50.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi159", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:29:50.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:50.540 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:50 smithi159 ceph-mon[67002]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:50.791 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:50 smithi159 ceph-mon[67002]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 685 op/s 2024-03-30T13:29:50.791 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:50 smithi159 ceph-mon[67002]: Reconfiguring crash.smithi159 (monmap changed)... 2024-03-30T13:29:50.791 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:50 smithi159 ceph-mon[67002]: Reconfiguring daemon crash.smithi159 on smithi159 2024-03-30T13:29:50.792 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:50 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1642334541' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:29:50.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 184462 11.69 MB/sec execute 252 sec latency 44.326 ms 2024-03-30T13:29:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:50 smithi187 ceph-mon[53604]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:29:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:50 smithi187 ceph-mon[53604]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 685 op/s 2024-03-30T13:29:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:50 smithi187 ceph-mon[53604]: Reconfiguring crash.smithi159 (monmap changed)... 2024-03-30T13:29:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:50 smithi187 ceph-mon[53604]: Reconfiguring daemon crash.smithi159 on smithi159 2024-03-30T13:29:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:50 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1642334541' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:29:51.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 184719 11.67 MB/sec execute 253 sec latency 44.136 ms 2024-03-30T13:29:52.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:52 smithi159 ceph-mon[67002]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 685 op/s 2024-03-30T13:29:52.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:52 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:52.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:52 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:52.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:52 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-30T13:29:52.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:52 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:52.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 184970 11.64 MB/sec execute 254 sec latency 54.872 ms 2024-03-30T13:29:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:52 smithi187 ceph-mon[53604]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 685 op/s 2024-03-30T13:29:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:52 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:52 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:52 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-30T13:29:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:52 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:53.629 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:53 smithi159 ceph-mon[67002]: Reconfiguring osd.0 (monmap changed)... 2024-03-30T13:29:53.629 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:53 smithi159 ceph-mon[67002]: Reconfiguring daemon osd.0 on smithi159 2024-03-30T13:29:53.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 185256 11.64 MB/sec execute 255 sec latency 52.977 ms 2024-03-30T13:29:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:53 smithi187 ceph-mon[53604]: Reconfiguring osd.0 (monmap changed)... 2024-03-30T13:29:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:53 smithi187 ceph-mon[53604]: Reconfiguring daemon osd.0 on smithi159 2024-03-30T13:29:54.644 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:54 smithi159 ceph-mon[67002]: pgmap v25: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.6 MiB/s wr, 657 op/s 2024-03-30T13:29:54.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 185757 11.64 MB/sec execute 256 sec latency 41.220 ms 2024-03-30T13:29:55.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:54 smithi187 ceph-mon[53604]: pgmap v25: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.6 MiB/s wr, 657 op/s 2024-03-30T13:29:55.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 186403 11.63 MB/sec execute 257 sec latency 58.766 ms 2024-03-30T13:29:56.703 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:56 smithi159 ceph-mon[67002]: pgmap v26: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 789 op/s 2024-03-30T13:29:56.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 186809 11.66 MB/sec execute 258 sec latency 29.708 ms 2024-03-30T13:29:57.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:56 smithi187 ceph-mon[53604]: pgmap v26: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 789 op/s 2024-03-30T13:29:57.896 INFO:teuthology.orchestra.run.smithi159.stdout: 5 187118 11.65 MB/sec execute 259 sec latency 75.515 ms 2024-03-30T13:29:58.238 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:58 smithi159 ceph-mon[67002]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.9 MiB/s wr, 492 op/s 2024-03-30T13:29:58.239 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:58.239 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:58.242 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:58 smithi159 ceph-mon[67002]: Reconfiguring osd.1 (monmap changed)... 2024-03-30T13:29:58.242 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-30T13:29:58.242 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:58.242 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:58 smithi159 ceph-mon[67002]: Reconfiguring daemon osd.1 on smithi159 2024-03-30T13:29:58.242 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:29:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:29:58.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:58 smithi187 ceph-mon[53604]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.9 MiB/s wr, 492 op/s 2024-03-30T13:29:58.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:58.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:29:58.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:58 smithi187 ceph-mon[53604]: Reconfiguring osd.1 (monmap changed)... 2024-03-30T13:29:58.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-30T13:29:58.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:29:58.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:58 smithi187 ceph-mon[53604]: Reconfiguring daemon osd.1 on smithi159 2024-03-30T13:29:58.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:29:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:29:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 187526 11.63 MB/sec execute 260 sec latency 61.224 ms 2024-03-30T13:29:59.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 187892 11.63 MB/sec execute 261 sec latency 38.071 ms 2024-03-30T13:30:00.249 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:00 smithi159 ceph-mon[67002]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 760 op/s 2024-03-30T13:30:00.249 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:00 smithi159 ceph-mon[67002]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:30:00.249 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:00 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:00.249 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:00 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:00.249 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:00 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-30T13:30:00.249 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:00 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:00.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:00 smithi187 ceph-mon[53604]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 760 op/s 2024-03-30T13:30:00.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:00 smithi187 ceph-mon[53604]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:30:00.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:00 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:00.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:00 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:00.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:00 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-30T13:30:00.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:00 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:00.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 188508 11.66 MB/sec execute 262 sec latency 32.902 ms 2024-03-30T13:30:01.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:01 smithi159 ceph-mon[67002]: Reconfiguring osd.2 (monmap changed)... 2024-03-30T13:30:01.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:01 smithi159 ceph-mon[67002]: Reconfiguring daemon osd.2 on smithi159 2024-03-30T13:30:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:01 smithi187 ceph-mon[53604]: Reconfiguring osd.2 (monmap changed)... 2024-03-30T13:30:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:01 smithi187 ceph-mon[53604]: Reconfiguring daemon osd.2 on smithi159 2024-03-30T13:30:01.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 189120 11.68 MB/sec execute 263 sec latency 32.354 ms 2024-03-30T13:30:02.257 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:02 smithi159 ceph-mon[67002]: pgmap v29: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 508 op/s 2024-03-30T13:30:02.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:02 smithi187 ceph-mon[53604]: pgmap v29: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 508 op/s 2024-03-30T13:30:02.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 189751 11.67 MB/sec execute 264 sec latency 27.028 ms 2024-03-30T13:30:03.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 190130 11.66 MB/sec execute 265 sec latency 48.365 ms 2024-03-30T13:30:04.279 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:04 smithi159 ceph-mon[67002]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 508 op/s 2024-03-30T13:30:04.279 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:04 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:04.279 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:04 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:04.279 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:04 smithi159 ceph-mon[67002]: Reconfiguring osd.3 (monmap changed)... 2024-03-30T13:30:04.279 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:04 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-30T13:30:04.279 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:04 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:04.279 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:04 smithi159 ceph-mon[67002]: Reconfiguring daemon osd.3 on smithi159 2024-03-30T13:30:04.323 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:04 smithi187 ceph-mon[53604]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 508 op/s 2024-03-30T13:30:04.323 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:04 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:04.323 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:04 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:04.323 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:04 smithi187 ceph-mon[53604]: Reconfiguring osd.3 (monmap changed)... 2024-03-30T13:30:04.323 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:04 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-30T13:30:04.323 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:04 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:04.324 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:04 smithi187 ceph-mon[53604]: Reconfiguring daemon osd.3 on smithi159 2024-03-30T13:30:04.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 190669 11.70 MB/sec execute 266 sec latency 30.804 ms 2024-03-30T13:30:05.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 191160 11.69 MB/sec execute 267 sec latency 55.768 ms 2024-03-30T13:30:06.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:06 smithi159 ceph-mon[67002]: pgmap v31: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 824 op/s 2024-03-30T13:30:06.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:06.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:06.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi159.jwwwqm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:30:06.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:06 smithi187 ceph-mon[53604]: pgmap v31: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 824 op/s 2024-03-30T13:30:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi159.jwwwqm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:30:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:06.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 191350 11.67 MB/sec execute 268 sec latency 62.374 ms 2024-03-30T13:30:07.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:07 smithi159 ceph-mon[67002]: Reconfiguring mds.foofs.smithi159.jwwwqm (monmap changed)... 2024-03-30T13:30:07.255 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:07 smithi159 ceph-mon[67002]: Reconfiguring daemon mds.foofs.smithi159.jwwwqm on smithi159 2024-03-30T13:30:07.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:07 smithi187 ceph-mon[53604]: Reconfiguring mds.foofs.smithi159.jwwwqm (monmap changed)... 2024-03-30T13:30:07.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:07 smithi187 ceph-mon[53604]: Reconfiguring daemon mds.foofs.smithi159.jwwwqm on smithi159 2024-03-30T13:30:07.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 191619 11.67 MB/sec execute 269 sec latency 49.712 ms 2024-03-30T13:30:08.279 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:08 smithi159 ceph-mon[67002]: pgmap v32: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.7 MiB/s wr, 583 op/s 2024-03-30T13:30:08.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:08 smithi187 ceph-mon[53604]: pgmap v32: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.7 MiB/s wr, 583 op/s 2024-03-30T13:30:08.897 INFO:teuthology.orchestra.run.smithi159.stdout: 5 191772 11.64 MB/sec execute 270 sec latency 82.902 ms 2024-03-30T13:30:09.319 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:09.319 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:09.320 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:09 smithi159 ceph-mon[67002]: Reconfiguring crash.smithi187 (monmap changed)... 2024-03-30T13:30:09.320 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:30:09.320 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:09.320 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:09 smithi159 ceph-mon[67002]: Reconfiguring daemon crash.smithi187 on smithi187 2024-03-30T13:30:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:09 smithi187 ceph-mon[53604]: Reconfiguring crash.smithi187 (monmap changed)... 2024-03-30T13:30:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:30:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:09 smithi187 ceph-mon[53604]: Reconfiguring daemon crash.smithi187 on smithi187 2024-03-30T13:30:09.899 INFO:teuthology.orchestra.run.smithi159.stdout: 5 192093 11.62 MB/sec execute 271 sec latency 67.676 ms 2024-03-30T13:30:10.334 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:10 smithi159 ceph-mon[67002]: pgmap v33: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 803 op/s 2024-03-30T13:30:10.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:10.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:10.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:30:10.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:30:10.335 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:10.639 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:10 smithi187 ceph-mon[53604]: pgmap v33: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 803 op/s 2024-03-30T13:30:10.639 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:10.639 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:10.639 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi187.jpkaud", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-30T13:30:10.639 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-30T13:30:10.639 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: 5 192427 11.62 MB/sec execute 272 sec latency 55.018 ms 2024-03-30T13:30:11.366 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:11 smithi159 ceph-mon[67002]: Reconfiguring mgr.smithi187.jpkaud (monmap changed)... 2024-03-30T13:30:11.366 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:11 smithi159 ceph-mon[67002]: Reconfiguring daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:30:11.617 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:11 smithi187 ceph-mon[53604]: Reconfiguring mgr.smithi187.jpkaud (monmap changed)... 2024-03-30T13:30:11.618 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:11 smithi187 ceph-mon[53604]: Reconfiguring daemon mgr.smithi187.jpkaud on smithi187 2024-03-30T13:30:11.907 INFO:teuthology.orchestra.run.smithi159.stdout: 5 192694 11.60 MB/sec execute 273 sec latency 38.726 ms 2024-03-30T13:30:12.381 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:12 smithi159 ceph-mon[67002]: pgmap v34: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.7 MiB/s wr, 535 op/s 2024-03-30T13:30:12.382 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:12.382 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:12.382 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:30:12.382 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:30:12.382 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:12.542 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:12 smithi187 ceph-mon[53604]: pgmap v34: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.7 MiB/s wr, 535 op/s 2024-03-30T13:30:12.542 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:12.542 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:12.542 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-30T13:30:12.542 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-30T13:30:12.542 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:12.908 INFO:teuthology.orchestra.run.smithi159.stdout: 5 192978 11.57 MB/sec execute 274 sec latency 65.544 ms 2024-03-30T13:30:13.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:13 smithi159 ceph-mon[67002]: Reconfiguring mon.smithi187 (monmap changed)... 2024-03-30T13:30:13.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:13 smithi159 ceph-mon[67002]: Reconfiguring daemon mon.smithi187 on smithi187 2024-03-30T13:30:13.434 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:30:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:13 smithi187 ceph-mon[53604]: Reconfiguring mon.smithi187 (monmap changed)... 2024-03-30T13:30:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:13 smithi187 ceph-mon[53604]: Reconfiguring daemon mon.smithi187 on smithi187 2024-03-30T13:30:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:30:13.908 INFO:teuthology.orchestra.run.smithi159.stdout: 5 193473 11.57 MB/sec execute 275 sec latency 70.821 ms 2024-03-30T13:30:14.445 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:14 smithi159 ceph-mon[67002]: pgmap v35: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.7 MiB/s wr, 535 op/s 2024-03-30T13:30:14.446 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:14 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:14.446 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:14 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:14.446 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:14 smithi159 ceph-mon[67002]: Reconfiguring osd.4 (monmap changed)... 2024-03-30T13:30:14.446 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:14 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-30T13:30:14.446 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:14 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:14.446 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:14 smithi159 ceph-mon[67002]: Reconfiguring daemon osd.4 on smithi187 2024-03-30T13:30:14.676 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:14 smithi187 ceph-mon[53604]: pgmap v35: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.7 MiB/s wr, 535 op/s 2024-03-30T13:30:14.676 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:14 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:14.677 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:14 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:14.677 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:14 smithi187 ceph-mon[53604]: Reconfiguring osd.4 (monmap changed)... 2024-03-30T13:30:14.677 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:14 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-30T13:30:14.677 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:14 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:14.677 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:14 smithi187 ceph-mon[53604]: Reconfiguring daemon osd.4 on smithi187 2024-03-30T13:30:14.908 INFO:teuthology.orchestra.run.smithi159.stdout: 5 193810 11.56 MB/sec execute 276 sec latency 34.092 ms 2024-03-30T13:30:15.908 INFO:teuthology.orchestra.run.smithi159.stdout: 5 194284 11.58 MB/sec execute 277 sec latency 37.782 ms 2024-03-30T13:30:16.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:16 smithi159 ceph-mon[67002]: pgmap v36: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 723 op/s 2024-03-30T13:30:16.208 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:16 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:16.208 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:16 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:16.208 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:16 smithi159 ceph-mon[67002]: Reconfiguring osd.5 (monmap changed)... 2024-03-30T13:30:16.208 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:16 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-30T13:30:16.208 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:16 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:16.208 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:16 smithi159 ceph-mon[67002]: Reconfiguring daemon osd.5 on smithi187 2024-03-30T13:30:16.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:16 smithi187 ceph-mon[53604]: pgmap v36: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 723 op/s 2024-03-30T13:30:16.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:16 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:16.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:16 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:16.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:16 smithi187 ceph-mon[53604]: Reconfiguring osd.5 (monmap changed)... 2024-03-30T13:30:16.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:16 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-30T13:30:16.407 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:16 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:16.408 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:16 smithi187 ceph-mon[53604]: Reconfiguring daemon osd.5 on smithi187 2024-03-30T13:30:16.908 INFO:teuthology.orchestra.run.smithi159.stdout: 5 194713 11.57 MB/sec execute 278 sec latency 49.887 ms 2024-03-30T13:30:17.909 INFO:teuthology.orchestra.run.smithi159.stdout: 5 194919 11.56 MB/sec execute 279 sec latency 51.264 ms 2024-03-30T13:30:18.263 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:18 smithi159 ceph-mon[67002]: pgmap v37: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-03-30T13:30:18.263 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:18.263 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:18.263 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:18 smithi159 ceph-mon[67002]: Reconfiguring osd.6 (monmap changed)... 2024-03-30T13:30:18.264 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-30T13:30:18.264 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:18.264 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:18 smithi159 ceph-mon[67002]: Reconfiguring daemon osd.6 on smithi187 2024-03-30T13:30:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:18 smithi187 ceph-mon[53604]: pgmap v37: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-03-30T13:30:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:18 smithi187 ceph-mon[53604]: Reconfiguring osd.6 (monmap changed)... 2024-03-30T13:30:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-30T13:30:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:18 smithi187 ceph-mon[53604]: Reconfiguring daemon osd.6 on smithi187 2024-03-30T13:30:18.909 INFO:teuthology.orchestra.run.smithi159.stdout: 5 195209 11.56 MB/sec execute 280 sec latency 51.379 ms 2024-03-30T13:30:19.909 INFO:teuthology.orchestra.run.smithi159.stdout: 5 195590 11.55 MB/sec execute 281 sec latency 52.389 ms 2024-03-30T13:30:20.093 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:30:20.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:20 smithi159 ceph-mon[67002]: pgmap v38: 97 pgs: 97 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-03-30T13:30:20.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:20.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:20.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:20 smithi159 ceph-mon[67002]: Reconfiguring osd.7 (monmap changed)... 2024-03-30T13:30:20.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-30T13:30:20.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:20.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:20 smithi159 ceph-mon[67002]: Reconfiguring daemon osd.7 on smithi187 2024-03-30T13:30:20.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:20 smithi187 ceph-mon[53604]: pgmap v38: 97 pgs: 97 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-03-30T13:30:20.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:20.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:20.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:20 smithi187 ceph-mon[53604]: Reconfiguring osd.7 (monmap changed)... 2024-03-30T13:30:20.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-30T13:30:20.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:20.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:20 smithi187 ceph-mon[53604]: Reconfiguring daemon osd.7 on smithi187 2024-03-30T13:30:20.909 INFO:teuthology.orchestra.run.smithi159.stdout: 5 195997 11.54 MB/sec execute 282 sec latency 56.620 ms 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (3m) 56s ago 13m 18.3M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (13m) 56s ago 13m 6409k - 17.2.0 e1d6a67b021e 599a045262ca 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (12m) 43s ago 12m 6405k - 17.2.0 e1d6a67b021e a819442e2554 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 unknown 56s ago 12m - - 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (7m) 56s ago 7m 37.9M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (7m) 43s ago 7m 16.6M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (2m) 56s ago 14m 476M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (101s) 43s ago 12m 448M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (79s) 56s ago 14m 30.6M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:30:21.035 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (47s) 43s ago 12m 24.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (6m) 56s ago 6m 275M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (6m) 43s ago 6m 86.0M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (3m) 56s ago 12m 12.7M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (2m) 43s ago 12m 19.9M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (11m) 56s ago 11m 326M 1936M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (11m) 56s ago 11m 515M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (10m) 56s ago 10m 429M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (10m) 56s ago 10m 520M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (9m) 43s ago 9m 539M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (9m) 43s ago 9m 407M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (9m) 43s ago 9m 567M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (8m) 43s ago 8m 434M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:30:21.036 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (108s) 56s ago 11m 55.5M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:30:21.574 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-30T13:30:21.575 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 4 2024-03-30T13:30:21.576 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:30:21.576 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:30:21.610 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:21 smithi159 ceph-mon[67002]: from='client.34174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:21.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:21 smithi159 ceph-mon[67002]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:21.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:21.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:21.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi187.bkvdqd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:30:21.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:21 smithi187 ceph-mon[53604]: from='client.34174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:21 smithi187 ceph-mon[53604]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi187.bkvdqd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:30:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:21.909 INFO:teuthology.orchestra.run.smithi159.stdout: 5 196284 11.53 MB/sec execute 283 sec latency 44.488 ms 2024-03-30T13:30:22.153 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:30:22.153 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:30:22.153 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:30:22.153 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:30:22.153 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:30:22.154 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:30:22.154 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr" 2024-03-30T13:30:22.154 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:30:22.154 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "4/23 daemons upgraded", 2024-03-30T13:30:22.154 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading mon daemons", 2024-03-30T13:30:22.154 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:30:22.154 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:30:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:22 smithi187 ceph-mon[53604]: pgmap v39: 97 pgs: 97 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 414 op/s 2024-03-30T13:30:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:22 smithi187 ceph-mon[53604]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:22 smithi187 ceph-mon[53604]: Reconfiguring mds.foofs.smithi187.bkvdqd (monmap changed)... 2024-03-30T13:30:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:22 smithi187 ceph-mon[53604]: Reconfiguring daemon mds.foofs.smithi187.bkvdqd on smithi187 2024-03-30T13:30:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:22 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/4116202195' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:22.616 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:22 smithi159 ceph-mon[67002]: pgmap v39: 97 pgs: 97 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 414 op/s 2024-03-30T13:30:22.616 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:22 smithi159 ceph-mon[67002]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:22.617 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:22 smithi159 ceph-mon[67002]: Reconfiguring mds.foofs.smithi187.bkvdqd (monmap changed)... 2024-03-30T13:30:22.617 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:22 smithi159 ceph-mon[67002]: Reconfiguring daemon mds.foofs.smithi187.bkvdqd on smithi187 2024-03-30T13:30:22.617 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:22 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/4116202195' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:22.793 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:30:22.793 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:30:22.794 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:30:22.909 INFO:teuthology.orchestra.run.smithi159.stdout: 5 196768 11.52 MB/sec execute 284 sec latency 45.530 ms 2024-03-30T13:30:23.624 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='client.34190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/730463119' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi159"}]: dispatch 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi159"}]': finished 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi187"}]: dispatch 2024-03-30T13:30:23.625 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi187"}]': finished 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='client.34190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/730463119' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi159"}]: dispatch 2024-03-30T13:30:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi159"}]': finished 2024-03-30T13:30:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi187"}]: dispatch 2024-03-30T13:30:23.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi187"}]': finished 2024-03-30T13:30:23.912 INFO:teuthology.orchestra.run.smithi159.stdout: 5 197203 11.52 MB/sec execute 285 sec latency 33.399 ms 2024-03-30T13:30:24.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:24 smithi159 ceph-mon[67002]: pgmap v40: 97 pgs: 97 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 414 op/s 2024-03-30T13:30:24.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:24 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all mon 2024-03-30T13:30:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:24 smithi187 ceph-mon[53604]: pgmap v40: 97 pgs: 97 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 414 op/s 2024-03-30T13:30:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:24 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all mon 2024-03-30T13:30:24.912 INFO:teuthology.orchestra.run.smithi159.stdout: 5 197673 11.54 MB/sec execute 286 sec latency 52.245 ms 2024-03-30T13:30:25.912 INFO:teuthology.orchestra.run.smithi159.stdout: 5 198221 11.54 MB/sec execute 287 sec latency 69.405 ms 2024-03-30T13:30:26.400 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:26 smithi159 ceph-mon[67002]: pgmap v41: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.3 MiB/s wr, 653 op/s 2024-03-30T13:30:26.400 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:26.401 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi159", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:30:26.401 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:26 smithi187 ceph-mon[53604]: pgmap v41: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.3 MiB/s wr, 653 op/s 2024-03-30T13:30:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:26 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:26 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi159", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:30:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:26 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:26.912 INFO:teuthology.orchestra.run.smithi159.stdout: 5 198499 11.53 MB/sec execute 288 sec latency 41.278 ms 2024-03-30T13:30:27.411 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:27 smithi159 ceph-mon[67002]: Upgrade: Updating crash.smithi159 (1/2) 2024-03-30T13:30:27.412 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:27 smithi159 ceph-mon[67002]: Deploying daemon crash.smithi159 on smithi159 2024-03-30T13:30:27.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:27 smithi187 ceph-mon[53604]: Upgrade: Updating crash.smithi159 (1/2) 2024-03-30T13:30:27.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:27 smithi187 ceph-mon[53604]: Deploying daemon crash.smithi159 on smithi159 2024-03-30T13:30:27.912 INFO:teuthology.orchestra.run.smithi159.stdout: 5 198877 11.53 MB/sec execute 289 sec latency 43.031 ms 2024-03-30T13:30:28.425 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:28 smithi159 ceph-mon[67002]: pgmap v42: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-03-30T13:30:28.425 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:30:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:28 smithi187 ceph-mon[53604]: pgmap v42: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-03-30T13:30:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:30:28.912 INFO:teuthology.orchestra.run.smithi159.stdout: 5 199243 11.52 MB/sec execute 290 sec latency 48.248 ms 2024-03-30T13:30:29.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 199593 11.52 MB/sec execute 291 sec latency 46.930 ms 2024-03-30T13:30:30.471 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:30 smithi159 ceph-mon[67002]: pgmap v43: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 716 op/s 2024-03-30T13:30:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:30 smithi187 ceph-mon[53604]: pgmap v43: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 716 op/s 2024-03-30T13:30:30.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 199828 11.50 MB/sec execute 292 sec latency 69.664 ms 2024-03-30T13:30:31.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 200176 11.49 MB/sec execute 293 sec latency 59.142 ms 2024-03-30T13:30:32.542 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:32 smithi159 ceph-mon[67002]: pgmap v44: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 490 op/s 2024-03-30T13:30:32.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:32 smithi187 ceph-mon[53604]: pgmap v44: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 490 op/s 2024-03-30T13:30:32.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 200496 11.47 MB/sec execute 294 sec latency 61.184 ms 2024-03-30T13:30:33.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 200734 11.45 MB/sec execute 295 sec latency 41.835 ms 2024-03-30T13:30:34.571 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:34 smithi187 ceph-mon[53604]: pgmap v45: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 490 op/s 2024-03-30T13:30:34.572 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:34.572 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:34.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:34 smithi159 ceph-mon[67002]: pgmap v45: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 490 op/s 2024-03-30T13:30:34.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:34.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:34.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 201027 11.45 MB/sec execute 296 sec latency 51.678 ms 2024-03-30T13:30:35.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 201332 11.45 MB/sec execute 297 sec latency 66.454 ms 2024-03-30T13:30:36.107 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:36 smithi159 ceph-mon[67002]: pgmap v46: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 681 op/s 2024-03-30T13:30:36.107 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:36 smithi159 ceph-mon[67002]: Upgrade: Updating crash.smithi187 (2/2) 2024-03-30T13:30:36.108 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:36.108 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:30:36.108 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:36.108 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:36 smithi159 ceph-mon[67002]: Deploying daemon crash.smithi187 on smithi187 2024-03-30T13:30:36.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:36 smithi187 ceph-mon[53604]: pgmap v46: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 681 op/s 2024-03-30T13:30:36.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:36 smithi187 ceph-mon[53604]: Upgrade: Updating crash.smithi187 (2/2) 2024-03-30T13:30:36.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:36.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi187", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-30T13:30:36.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:36.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:36 smithi187 ceph-mon[53604]: Deploying daemon crash.smithi187 on smithi187 2024-03-30T13:30:36.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 201800 11.44 MB/sec execute 298 sec latency 39.536 ms 2024-03-30T13:30:37.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 202033 11.43 MB/sec execute 299 sec latency 44.415 ms 2024-03-30T13:30:38.229 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:38 smithi159 ceph-mon[67002]: pgmap v47: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-03-30T13:30:38.295 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:38 smithi187 ceph-mon[53604]: pgmap v47: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-03-30T13:30:38.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 202599 11.45 MB/sec execute 300 sec latency 43.463 ms 2024-03-30T13:30:39.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 203147 11.45 MB/sec execute 301 sec latency 42.298 ms 2024-03-30T13:30:40.494 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:40 smithi159 ceph-mon[67002]: pgmap v48: 97 pgs: 97 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.6 MiB/s wr, 639 op/s 2024-03-30T13:30:40.494 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:40 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:40.494 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:40 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:40.495 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:40 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:30:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:40 smithi187 ceph-mon[53604]: pgmap v48: 97 pgs: 97 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.6 MiB/s wr, 639 op/s 2024-03-30T13:30:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:40 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:40 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:40 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:30:40.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 203644 11.46 MB/sec execute 302 sec latency 45.072 ms 2024-03-30T13:30:41.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 204238 11.46 MB/sec execute 303 sec latency 35.579 ms 2024-03-30T13:30:42.270 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:42 smithi159 ceph-mon[67002]: pgmap v49: 97 pgs: 97 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 388 op/s 2024-03-30T13:30:42.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:42 smithi187 ceph-mon[53604]: pgmap v49: 97 pgs: 97 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 388 op/s 2024-03-30T13:30:42.914 INFO:teuthology.orchestra.run.smithi159.stdout: 5 204659 11.48 MB/sec execute 304 sec latency 36.411 ms 2024-03-30T13:30:43.283 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:30:43.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:30:43.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 205079 11.47 MB/sec execute 305 sec latency 56.248 ms 2024-03-30T13:30:44.293 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:44 smithi159 ceph-mon[67002]: pgmap v50: 97 pgs: 97 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 388 op/s 2024-03-30T13:30:44.293 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:44 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:44.293 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:44 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:44.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:44 smithi187 ceph-mon[53604]: pgmap v50: 97 pgs: 97 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 388 op/s 2024-03-30T13:30:44.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:44 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:44.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:44 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:44.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 205447 11.46 MB/sec execute 306 sec latency 41.711 ms 2024-03-30T13:30:45.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 205672 11.45 MB/sec execute 307 sec latency 40.553 ms 2024-03-30T13:30:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:46 smithi187 ceph-mon[53604]: pgmap v51: 97 pgs: 97 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 704 op/s 2024-03-30T13:30:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:46 smithi187 ceph-mon[53604]: mgrmap e43: smithi159.fnasly(active, since 92s), standbys: smithi187.jpkaud 2024-03-30T13:30:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:46.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:46.326 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:46 smithi159 ceph-mon[67002]: pgmap v51: 97 pgs: 97 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 704 op/s 2024-03-30T13:30:46.327 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:46 smithi159 ceph-mon[67002]: mgrmap e43: smithi159.fnasly(active, since 92s), standbys: smithi187.jpkaud 2024-03-30T13:30:46.327 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:46.327 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:46.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 206078 11.45 MB/sec execute 308 sec latency 51.327 ms 2024-03-30T13:30:47.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 206498 11.45 MB/sec execute 309 sec latency 42.373 ms 2024-03-30T13:30:48.352 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:48 smithi159 ceph-mon[67002]: pgmap v52: 97 pgs: 97 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-03-30T13:30:48.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:48 smithi187 ceph-mon[53604]: pgmap v52: 97 pgs: 97 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-03-30T13:30:48.913 INFO:teuthology.orchestra.run.smithi159.stdout: 5 206766 11.44 MB/sec execute 310 sec latency 56.672 ms 2024-03-30T13:30:49.624 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:49.624 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:49.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:49.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:49.914 INFO:teuthology.orchestra.run.smithi159.stdout: 5 207115 11.44 MB/sec execute 311 sec latency 41.340 ms 2024-03-30T13:30:50.635 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:50 smithi159 ceph-mon[67002]: pgmap v53: 97 pgs: 97 active+clean; 352 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 734 op/s 2024-03-30T13:30:50.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:50 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:50.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:50 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:50.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:50 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:50.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:50 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:50.914 INFO:teuthology.orchestra.run.smithi159.stdout: 5 207569 11.42 MB/sec execute 312 sec latency 41.978 ms 2024-03-30T13:30:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:50 smithi187 ceph-mon[53604]: pgmap v53: 97 pgs: 97 active+clean; 352 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 734 op/s 2024-03-30T13:30:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:50 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:50 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:50 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:50 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:51.914 INFO:teuthology.orchestra.run.smithi159.stdout: 5 207964 11.42 MB/sec execute 313 sec latency 41.154 ms 2024-03-30T13:30:52.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:52 smithi159 ceph-mon[67002]: pgmap v54: 97 pgs: 97 active+clean; 352 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 536 op/s 2024-03-30T13:30:52.914 INFO:teuthology.orchestra.run.smithi159.stdout: 5 208503 11.44 MB/sec execute 314 sec latency 33.110 ms 2024-03-30T13:30:53.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:52 smithi187 ceph-mon[53604]: pgmap v54: 97 pgs: 97 active+clean; 352 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 536 op/s 2024-03-30T13:30:53.384 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:30:53.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 208975 11.44 MB/sec execute 315 sec latency 41.025 ms 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (4m) 5s ago 13m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (21s) 5s ago 13m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (15s) 10s ago 12m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 5s ago 13m - - 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (7m) 5s ago 7m 38.3M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (7m) 10s ago 7m 16.8M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (3m) 5s ago 14m 501M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (2m) 10s ago 12m 449M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (113s) 5s ago 15m 41.0M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:30:54.310 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (80s) 10s ago 12m 34.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (7m) 5s ago 7m 281M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (7m) 10s ago 7m 86.1M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (3m) 5s ago 13m 10.3M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (3m) 10s ago 12m 18.3M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (12m) 5s ago 12m 437M 1936M 17.2.0 e1d6a67b021e 4dcb087adc27 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (11m) 5s ago 11m 360M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (11m) 5s ago 11m 535M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (10m) 5s ago 10m 350M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (10m) 10s ago 10m 595M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (9m) 10s ago 9m 465M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (9m) 10s ago 9m 378M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (9m) 10s ago 9m 538M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:30:54.311 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (2m) 5s ago 12m 58.4M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:30:54.688 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:54 smithi159 ceph-mon[67002]: pgmap v55: 97 pgs: 97 active+clean; 352 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 536 op/s 2024-03-30T13:30:54.689 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:54 smithi159 ceph-mon[67002]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:30:54.814 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:30:54.815 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:30:54.815 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:30:54.815 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:30:54.815 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-30T13:30:54.815 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 4 2024-03-30T13:30:54.815 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:30:54.815 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:30:54.914 INFO:teuthology.orchestra.run.smithi159.stdout: 5 209149 11.42 MB/sec execute 316 sec latency 58.458 ms 2024-03-30T13:30:55.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:54 smithi187 ceph-mon[53604]: pgmap v55: 97 pgs: 97 active+clean; 352 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 536 op/s 2024-03-30T13:30:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:54 smithi187 ceph-mon[53604]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:55.405 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "6/23 daemons upgraded", 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading crash daemons", 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:30:55.406 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:30:55.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:55 smithi159 ceph-mon[67002]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:55.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:55 smithi159 ceph-mon[67002]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:55.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:55 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/308425198' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:55.902 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:30:55.903 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:30:55.903 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in unknown state 2024-03-30T13:30:55.914 INFO:teuthology.orchestra.run.smithi159.stdout: 5 209422 11.41 MB/sec execute 317 sec latency 39.402 ms 2024-03-30T13:30:56.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:55 smithi187 ceph-mon[53604]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:55 smithi187 ceph-mon[53604]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:55 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/308425198' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:56.734 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: pgmap v56: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 791 op/s 2024-03-30T13:30:56.734 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi159"}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi159"}]': finished 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi187"}]: dispatch 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi187"}]': finished 2024-03-30T13:30:56.735 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-30T13:30:56.736 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:56 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/4235893719' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:30:56.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 209704 11.40 MB/sec execute 318 sec latency 89.593 ms 2024-03-30T13:30:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: pgmap v56: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 791 op/s 2024-03-30T13:30:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:30:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:30:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi159"}]: dispatch 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi159"}]': finished 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi187"}]: dispatch 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi187"}]': finished 2024-03-30T13:30:57.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-30T13:30:57.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:56 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/4235893719' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:30:57.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:57 smithi159 ceph-mon[67002]: pgmap v57: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 530 op/s 2024-03-30T13:30:57.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:57 smithi159 ceph-mon[67002]: pgmap v58: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.5 MiB/s wr, 652 op/s 2024-03-30T13:30:57.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:57 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all crash 2024-03-30T13:30:57.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:57 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-30T13:30:57.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:57 smithi159 ceph-mon[67002]: Upgrade: osd.0 is safe to restart 2024-03-30T13:30:57.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:57 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:30:57.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 209992 11.38 MB/sec execute 319 sec latency 56.067 ms 2024-03-30T13:30:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:57 smithi187 ceph-mon[53604]: pgmap v57: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 530 op/s 2024-03-30T13:30:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:57 smithi187 ceph-mon[53604]: pgmap v58: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.5 MiB/s wr, 652 op/s 2024-03-30T13:30:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:57 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all crash 2024-03-30T13:30:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:57 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-30T13:30:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:57 smithi187 ceph-mon[53604]: Upgrade: osd.0 is safe to restart 2024-03-30T13:30:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:57 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:30:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 210226 11.37 MB/sec execute 320 sec latency 56.643 ms 2024-03-30T13:30:59.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:59 smithi159 ceph-mon[67002]: pgmap v59: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 349 op/s 2024-03-30T13:30:59.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:59 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:30:59.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:59 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-30T13:30:59.797 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:30:59 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:30:59.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 210437 11.36 MB/sec execute 321 sec latency 54.024 ms 2024-03-30T13:31:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:59 smithi187 ceph-mon[53604]: pgmap v59: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 349 op/s 2024-03-30T13:31:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:59 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:59 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-30T13:31:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:30:59 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:31:00.805 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:00 smithi159 ceph-mon[67002]: Upgrade: Updating osd.0 2024-03-30T13:31:00.806 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:00 smithi159 ceph-mon[67002]: Deploying daemon osd.0 on smithi159 2024-03-30T13:31:00.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 210717 11.35 MB/sec execute 322 sec latency 64.561 ms 2024-03-30T13:31:01.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:00 smithi187 ceph-mon[53604]: Upgrade: Updating osd.0 2024-03-30T13:31:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:00 smithi187 ceph-mon[53604]: Deploying daemon osd.0 on smithi159 2024-03-30T13:31:01.827 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:01 smithi159 ceph-mon[67002]: pgmap v60: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 582 op/s 2024-03-30T13:31:01.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 211095 11.32 MB/sec execute 323 sec latency 53.310 ms 2024-03-30T13:31:02.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:01 smithi187 ceph-mon[53604]: pgmap v60: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 582 op/s 2024-03-30T13:31:02.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 211415 11.32 MB/sec execute 324 sec latency 47.905 ms 2024-03-30T13:31:03.609 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:03 smithi159 ceph-mon[67002]: pgmap v61: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 582 op/s 2024-03-30T13:31:03.865 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:03 smithi159 systemd[1]: Stopping Ceph osd.0 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:31:03.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 211799 11.33 MB/sec execute 325 sec latency 38.181 ms 2024-03-30T13:31:04.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:03 smithi187 ceph-mon[53604]: pgmap v61: 97 pgs: 97 active+clean; 348 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 582 op/s 2024-03-30T13:31:04.369 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:04 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[35130]: 2024-03-30T13:31:04.273+0000 7f9e0b817700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:31:04.369 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:04 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[35130]: 2024-03-30T13:31:04.273+0000 7f9e0b817700 -1 osd.0 73 *** Got signal Terminated *** 2024-03-30T13:31:04.369 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:04 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[35130]: 2024-03-30T13:31:04.273+0000 7f9e0b817700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:31:04.648 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:04 smithi159 ceph-mon[67002]: osd.0 marked itself down and dead 2024-03-30T13:31:04.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 212089 11.31 MB/sec execute 326 sec latency 288.825 ms 2024-03-30T13:31:04.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:04 smithi187 ceph-mon[53604]: osd.0 marked itself down and dead 2024-03-30T13:31:05.587 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:05 smithi159 podman[72999]: 2024-03-30 13:31:05.288821395 +0000 UTC m=+1.213104300 container died 4dcb087adc270a94696402c52230e13daa137cb9d4fba567cf8953e20d98cb4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vcs-type=git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, ceph=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, version=8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-30T13:31:05.837 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:05 smithi159 podman[72999]: 2024-03-30 13:31:05.701533909 +0000 UTC m=+1.625816813 container cleanup 4dcb087adc270a94696402c52230e13daa137cb9d4fba567cf8953e20d98cb4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, ceph=True, distribution-scope=public, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , architecture=x86_64) 2024-03-30T13:31:05.837 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:05 smithi159 bash[72999]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0 2024-03-30T13:31:05.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:05 smithi159 ceph-mon[67002]: pgmap v62: 97 pgs: 97 active+clean; 350 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 500 op/s 2024-03-30T13:31:05.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:05 smithi159 ceph-mon[67002]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:31:05.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:05 smithi159 ceph-mon[67002]: failed to encode map e74 with expected crc 2024-03-30T13:31:05.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:05 smithi159 ceph-mon[67002]: osdmap e74: 8 total, 7 up, 8 in 2024-03-30T13:31:05.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:05 smithi159 ceph-mon[67002]: failed to encode map e74 with expected crc 2024-03-30T13:31:05.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 212234 11.28 MB/sec execute 327 sec latency 1031.189 ms 2024-03-30T13:31:05.938 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:05 smithi187 ceph-mon[53604]: pgmap v62: 97 pgs: 97 active+clean; 350 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 500 op/s 2024-03-30T13:31:05.938 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:05 smithi187 ceph-mon[53604]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:31:05.938 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:05 smithi187 ceph-mon[53604]: failed to encode map e74 with expected crc 2024-03-30T13:31:05.938 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:05 smithi187 ceph-mon[53604]: osdmap e74: 8 total, 7 up, 8 in 2024-03-30T13:31:05.939 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:05 smithi187 ceph-mon[53604]: failed to encode map e74 with expected crc 2024-03-30T13:31:06.088 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:06 smithi159 podman[73016]: 2024-03-30 13:31:06.045485379 +0000 UTC m=+0.753902883 container remove 4dcb087adc270a94696402c52230e13daa137cb9d4fba567cf8953e20d98cb4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-03-30T13:31:06.591 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:06 smithi159 podman[73098]: 2024-03-30 13:31:06.489285109 +0000 UTC m=+0.025062340 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:31:06.848 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:06 smithi159 podman[73098]: 2024-03-30 13:31:06.615656728 +0000 UTC m=+0.151433954 container create 140774a0e216575b5a8f68f793fe904dced6a44c3c2a2c529163915afe2b585e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc) 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: failed to encode map e75 with expected crc 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: osdmap e75: 8 total, 7 up, 8 in 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: failed to encode map e75 with expected crc 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: failed to encode map e75 with expected crc 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: failed to encode map e75 with expected crc 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: failed to encode map e75 with expected crc 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: failed to encode map e75 with expected crc 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: failed to encode map e75 with expected crc 2024-03-30T13:31:06.849 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:06 smithi159 ceph-mon[67002]: failed to encode map e75 with expected crc 2024-03-30T13:31:06.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 212610 11.29 MB/sec execute 328 sec latency 36.458 ms 2024-03-30T13:31:07.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: failed to encode map e75 with expected crc 2024-03-30T13:31:07.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: osdmap e75: 8 total, 7 up, 8 in 2024-03-30T13:31:07.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: failed to encode map e75 with expected crc 2024-03-30T13:31:07.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: failed to encode map e75 with expected crc 2024-03-30T13:31:07.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: failed to encode map e75 with expected crc 2024-03-30T13:31:07.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: failed to encode map e75 with expected crc 2024-03-30T13:31:07.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: failed to encode map e75 with expected crc 2024-03-30T13:31:07.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: failed to encode map e75 with expected crc 2024-03-30T13:31:07.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:06 smithi187 ceph-mon[53604]: failed to encode map e75 with expected crc 2024-03-30T13:31:07.099 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:06 smithi159 podman[73098]: 2024-03-30 13:31:06.975344614 +0000 UTC m=+0.511121836 container init 140774a0e216575b5a8f68f793fe904dced6a44c3c2a2c529163915afe2b585e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-03-30T13:31:07.100 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:06 smithi159 podman[73098]: 2024-03-30 13:31:06.980318135 +0000 UTC m=+0.516095367 container start 140774a0e216575b5a8f68f793fe904dced6a44c3c2a2c529163915afe2b585e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-4e50bbc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-30T13:31:07.100 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:07 smithi159 podman[73098]: 2024-03-30 13:31:07.0839559 +0000 UTC m=+0.619733130 container attach 140774a0e216575b5a8f68f793fe904dced6a44c3c2a2c529163915afe2b585e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-03-30T13:31:07.623 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:07 smithi159 podman[73098]: 2024-03-30 13:31:07.398677223 +0000 UTC m=+0.934454447 container died 140774a0e216575b5a8f68f793fe904dced6a44c3c2a2c529163915afe2b585e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-03-30T13:31:07.873 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:07 smithi159 ceph-mon[67002]: pgmap v65: 97 pgs: 13 stale+active+clean, 84 active+clean; 350 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.6 MiB/s wr, 547 op/s 2024-03-30T13:31:07.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 212794 11.27 MB/sec execute 329 sec latency 200.293 ms 2024-03-30T13:31:08.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:07 smithi187 ceph-mon[53604]: pgmap v65: 97 pgs: 13 stale+active+clean, 84 active+clean; 350 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.6 MiB/s wr, 547 op/s 2024-03-30T13:31:08.892 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:08 smithi159 podman[73123]: 2024-03-30 13:31:08.887227506 +0000 UTC m=+1.482277076 container remove 140774a0e216575b5a8f68f793fe904dced6a44c3c2a2c529163915afe2b585e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:31:08.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 213194 11.27 MB/sec execute 330 sec latency 52.598 ms 2024-03-30T13:31:08.925 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:08 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.0.service: Deactivated successfully. 2024-03-30T13:31:08.925 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:08 smithi159 systemd[1]: Stopped Ceph osd.0 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:31:08.925 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:08 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.0.service: Consumed 3min 45.418s CPU time. 2024-03-30T13:31:09.376 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:09 smithi159 systemd[1]: Starting Ceph osd.0 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:31:09.629 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:09 smithi159 ceph-mon[67002]: pgmap v66: 97 pgs: 13 stale+active+clean, 84 active+clean; 350 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.9 MiB/s wr, 292 op/s 2024-03-30T13:31:09.629 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:09 smithi159 podman[73257]: 2024-03-30 13:31:09.402646714 +0000 UTC m=+0.024095233 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:31:09.630 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:09 smithi159 podman[73257]: 2024-03-30 13:31:09.568962504 +0000 UTC m=+0.190411050 container create 1bc619d545683fca1793de1ed2cf495717d1a27a9630eb770f824c7a13cee52e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:31:09.888 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:09 smithi159 podman[73257]: 2024-03-30 13:31:09.863766964 +0000 UTC m=+0.485215486 container init 1bc619d545683fca1793de1ed2cf495717d1a27a9630eb770f824c7a13cee52e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-30T13:31:09.888 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:09 smithi159 podman[73257]: 2024-03-30 13:31:09.87240436 +0000 UTC m=+0.493852875 container start 1bc619d545683fca1793de1ed2cf495717d1a27a9630eb770f824c7a13cee52e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-30T13:31:09.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 213576 11.25 MB/sec execute 331 sec latency 41.170 ms 2024-03-30T13:31:10.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:09 smithi187 ceph-mon[53604]: pgmap v66: 97 pgs: 13 stale+active+clean, 84 active+clean; 350 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.9 MiB/s wr, 292 op/s 2024-03-30T13:31:10.142 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 podman[73257]: 2024-03-30 13:31:10.021131347 +0000 UTC m=+0.642579865 container attach 1bc619d545683fca1793de1ed2cf495717d1a27a9630eb770f824c7a13cee52e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate[73270]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate[73270]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 bash[73257]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 bash[73257]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 bash[73257]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 bash[73257]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 bash[73257]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 bash[73257]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 bash[73257]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate[73270]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate[73270]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-30T13:31:10.670 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate[73270]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-30T13:31:10.671 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate[73270]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-30T13:31:10.671 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate[73270]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-30T13:31:10.671 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:10 smithi159 podman[73257]: 2024-03-30 13:31:10.496094467 +0000 UTC m=+1.117542982 container died 1bc619d545683fca1793de1ed2cf495717d1a27a9630eb770f824c7a13cee52e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-30T13:31:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 213864 11.26 MB/sec execute 332 sec latency 52.885 ms 2024-03-30T13:31:10.926 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-mon[67002]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 348 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 546 op/s; 225/1647 objects degraded (13.661%) 2024-03-30T13:31:10.926 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:10 smithi159 ceph-mon[67002]: Health check failed: Degraded data redundancy: 225/1647 objects degraded (13.661%), 29 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:10 smithi187 ceph-mon[53604]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 348 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 546 op/s; 225/1647 objects degraded (13.661%) 2024-03-30T13:31:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:10 smithi187 ceph-mon[53604]: Health check failed: Degraded data redundancy: 225/1647 objects degraded (13.661%), 29 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:11.436 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:11 smithi159 podman[73399]: 2024-03-30 13:31:11.396092546 +0000 UTC m=+0.894725247 container remove 1bc619d545683fca1793de1ed2cf495717d1a27a9630eb770f824c7a13cee52e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-activate, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-03-30T13:31:11.698 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:11 smithi159 podman[73457]: 2024-03-30 13:31:11.538563654 +0000 UTC m=+0.021240860 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:31:11.698 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:11 smithi159 podman[73457]: 2024-03-30 13:31:11.647124408 +0000 UTC m=+0.129801596 container create a21add6ba181b43ae9d607fc3b0216eb387dbf6eb5e24fa8b72e3b016d31c9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-03-30T13:31:11.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 214168 11.26 MB/sec execute 333 sec latency 48.186 ms 2024-03-30T13:31:12.203 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:11 smithi159 podman[73457]: 2024-03-30 13:31:11.990760849 +0000 UTC m=+0.473438059 container init a21add6ba181b43ae9d607fc3b0216eb387dbf6eb5e24fa8b72e3b016d31c9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0) 2024-03-30T13:31:12.204 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:11 smithi159 podman[73457]: 2024-03-30 13:31:11.996085648 +0000 UTC m=+0.478762837 container start a21add6ba181b43ae9d607fc3b0216eb387dbf6eb5e24fa8b72e3b016d31c9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326) 2024-03-30T13:31:12.204 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:12 smithi159 ceph-osd[73475]: -- 172.21.15.159:0/2552702210 <== mon.0 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56079658f680 con 0x560796559000 2024-03-30T13:31:12.459 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:12 smithi159 bash[73457]: a21add6ba181b43ae9d607fc3b0216eb387dbf6eb5e24fa8b72e3b016d31c9b8 2024-03-30T13:31:12.460 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:12 smithi159 systemd[1]: Started Ceph osd.0 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:31:12.710 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:12 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[73471]: 2024-03-30T13:31:12.574+0000 7f6bb83af7c0 -1 Falling back to public interface 2024-03-30T13:31:12.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 214605 11.24 MB/sec execute 334 sec latency 45.147 ms 2024-03-30T13:31:13.482 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:13 smithi159 ceph-mon[67002]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 348 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 MiB/s wr, 254 op/s; 225/1647 objects degraded (13.661%) 2024-03-30T13:31:13.483 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:13.483 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:13.483 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:31:13.483 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:13.483 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:31:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:13 smithi187 ceph-mon[53604]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 348 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 MiB/s wr, 254 op/s; 225/1647 objects degraded (13.661%) 2024-03-30T13:31:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:31:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:31:13.915 INFO:teuthology.orchestra.run.smithi159.stdout: 5 214935 11.23 MB/sec execute 335 sec latency 44.442 ms 2024-03-30T13:31:14.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 215275 11.23 MB/sec execute 336 sec latency 51.697 ms 2024-03-30T13:31:15.523 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:15 smithi159 ceph-mon[67002]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 499 op/s; 210/1695 objects degraded (12.389%) 2024-03-30T13:31:15.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:15 smithi187 ceph-mon[53604]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 499 op/s; 210/1695 objects degraded (12.389%) 2024-03-30T13:31:15.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 215685 11.23 MB/sec execute 337 sec latency 52.220 ms 2024-03-30T13:31:16.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 216209 11.23 MB/sec execute 338 sec latency 54.729 ms 2024-03-30T13:31:17.562 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:17 smithi159 ceph-mon[67002]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 450 op/s; 210/1695 objects degraded (12.389%) 2024-03-30T13:31:17.562 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:17 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 210/1695 objects degraded (12.389%), 29 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:17.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:17 smithi187 ceph-mon[53604]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 450 op/s; 210/1695 objects degraded (12.389%) 2024-03-30T13:31:17.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:17 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 210/1695 objects degraded (12.389%), 29 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:17.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 216496 11.22 MB/sec execute 339 sec latency 59.147 ms 2024-03-30T13:31:18.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 216991 11.23 MB/sec execute 340 sec latency 48.325 ms 2024-03-30T13:31:19.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:19 smithi159 ceph-mon[67002]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 377 op/s; 210/1695 objects degraded (12.389%) 2024-03-30T13:31:19.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:19 smithi187 ceph-mon[53604]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 377 op/s; 210/1695 objects degraded (12.389%) 2024-03-30T13:31:19.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 217361 11.23 MB/sec execute 341 sec latency 32.392 ms 2024-03-30T13:31:20.591 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:20 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[73471]: 2024-03-30T13:31:20.468+0000 7f6bb83af7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-03-30T13:31:20.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 217683 11.23 MB/sec execute 342 sec latency 41.194 ms 2024-03-30T13:31:21.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:21 smithi159 ceph-mon[67002]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 618 op/s; 229/1704 objects degraded (13.439%) 2024-03-30T13:31:21.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:21.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:21 smithi187 ceph-mon[53604]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 618 op/s; 229/1704 objects degraded (13.439%) 2024-03-30T13:31:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:21.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:21.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 218000 11.22 MB/sec execute 343 sec latency 39.983 ms 2024-03-30T13:31:22.105 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:21 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[73471]: 2024-03-30T13:31:21.990+0000 7f6bb83af7c0 -1 osd.0 73 log_to_monitors true 2024-03-30T13:31:22.671 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:22 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 229/1704 objects degraded (13.439%), 29 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:22.672 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:22 smithi159 ceph-mon[67002]: from='osd.0 [v2:172.21.15.159:6802/738332212,v1:172.21.15.159:6803/738332212]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-30T13:31:22.672 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:22 smithi159 ceph-mon[67002]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-30T13:31:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:22 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 229/1704 objects degraded (13.439%), 29 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:22 smithi187 ceph-mon[53604]: from='osd.0 [v2:172.21.15.159:6802/738332212,v1:172.21.15.159:6803/738332212]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-30T13:31:22.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:22 smithi187 ceph-mon[53604]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-30T13:31:22.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 218325 11.20 MB/sec execute 344 sec latency 146.585 ms 2024-03-30T13:31:23.192 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[73471]: 2024-03-30T13:31:23.016+0000 7f6ba57ff700 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: pgmap v73: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 449 op/s; 229/1704 objects degraded (13.439%) 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: failed to encode map e76 with expected crc 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: osdmap e76: 8 total, 7 up, 8 in 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: failed to encode map e76 with expected crc 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: from='osd.0 [v2:172.21.15.159:6802/738332212,v1:172.21.15.159:6803/738332212]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:23.463 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:23 smithi159 ceph-mon[67002]: from='osd.0 ' entity='osd.0' 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: pgmap v73: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 449 op/s; 229/1704 objects degraded (13.439%) 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: failed to encode map e76 with expected crc 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: osdmap e76: 8 total, 7 up, 8 in 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: failed to encode map e76 with expected crc 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: from='osd.0 [v2:172.21.15.159:6802/738332212,v1:172.21.15.159:6803/738332212]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:23.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:23 smithi187 ceph-mon[53604]: from='osd.0 ' entity='osd.0' 2024-03-30T13:31:23.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 218584 11.18 MB/sec execute 345 sec latency 38.369 ms 2024-03-30T13:31:24.477 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:24 smithi159 ceph-mon[67002]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:31:24.477 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:24 smithi159 ceph-mon[67002]: failed to encode map e77 with expected crc 2024-03-30T13:31:24.477 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:24 smithi159 ceph-mon[67002]: osd.0 [v2:172.21.15.159:6802/738332212,v1:172.21.15.159:6803/738332212] boot 2024-03-30T13:31:24.477 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:24 smithi159 ceph-mon[67002]: osdmap e77: 8 total, 8 up, 8 in 2024-03-30T13:31:24.477 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:24 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:31:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:24 smithi187 ceph-mon[53604]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:31:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:24 smithi187 ceph-mon[53604]: failed to encode map e77 with expected crc 2024-03-30T13:31:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:24 smithi187 ceph-mon[53604]: osd.0 [v2:172.21.15.159:6802/738332212,v1:172.21.15.159:6803/738332212] boot 2024-03-30T13:31:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:24 smithi187 ceph-mon[53604]: osdmap e77: 8 total, 8 up, 8 in 2024-03-30T13:31:24.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:24 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-30T13:31:24.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 218787 11.17 MB/sec execute 346 sec latency 789.337 ms 2024-03-30T13:31:25.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:25 smithi159 ceph-mon[67002]: pgmap v75: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 542 op/s; 213/1686 objects degraded (12.633%) 2024-03-30T13:31:25.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:25 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:25.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:25 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:25.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:25 smithi159 ceph-mon[67002]: osdmap e78: 8 total, 8 up, 8 in 2024-03-30T13:31:25.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:25 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:25.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:25 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:25 smithi187 ceph-mon[53604]: pgmap v75: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 542 op/s; 213/1686 objects degraded (12.633%) 2024-03-30T13:31:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:25 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:25 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:25 smithi187 ceph-mon[53604]: osdmap e78: 8 total, 8 up, 8 in 2024-03-30T13:31:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:25 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:25 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:25.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 218787 11.14 MB/sec execute 347 sec latency 1789.360 ms 2024-03-30T13:31:26.533 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:31:26.610 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: osdmap e79: 8 total, 8 up, 8 in 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:26.611 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:26 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e78 with expected crc 2024-03-30T13:31:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: osdmap e79: 8 total, 8 up, 8 in 2024-03-30T13:31:26.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:26.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:26 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:26.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 219197 11.15 MB/sec execute 348 sec latency 1971.902 ms 2024-03-30T13:31:27.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: pgmap v78: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 421 op/s; 213/1686 objects degraded (12.633%) 2024-03-30T13:31:27.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.619 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 213/1686 objects degraded (12.633%), 29 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: osdmap e80: 8 total, 8 up, 8 in 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e80 with expected crc 2024-03-30T13:31:27.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:27 smithi159 ceph-mon[67002]: failed to encode map e80 with expected crc 2024-03-30T13:31:27.704 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:31:27.704 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (4m) 6s ago 14m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:31:27.704 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (54s) 6s ago 14m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:31:27.704 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (48s) 43s ago 13m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:31:27.704 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 6s ago 13m - - 2024-03-30T13:31:27.704 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (8m) 6s ago 8m 38.4M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:31:27.704 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (8m) 43s ago 8m 16.8M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (3m) 6s ago 15m 502M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (2m) 43s ago 13m 449M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (2m) 6s ago 15m 43.8M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (114s) 43s ago 13m 34.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (8m) 6s ago 8m 282M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (7m) 43s ago 7m 86.1M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (4m) 6s ago 13m 10.4M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (3m) 43s ago 13m 18.3M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (16s) 6s ago 12m 12.7M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (12m) 6s ago 12m 376M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (11m) 6s ago 11m 562M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (11m) 6s ago 11m 384M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (10m) 43s ago 10m 595M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (10m) 43s ago 10m 465M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:31:27.705 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (10m) 43s ago 10m 378M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:31:27.706 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (9m) 43s ago 9m 538M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:31:27.706 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (2m) 6s ago 12m 62.9M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:31:27.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: pgmap v78: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 421 op/s; 213/1686 objects degraded (12.633%) 2024-03-30T13:31:27.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e79 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 213/1686 objects degraded (12.633%), 29 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: osdmap e80: 8 total, 8 up, 8 in 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e80 with expected crc 2024-03-30T13:31:27.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:27 smithi187 ceph-mon[53604]: failed to encode map e80 with expected crc 2024-03-30T13:31:27.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 219739 11.16 MB/sec execute 349 sec latency 820.675 ms 2024-03-30T13:31:28.370 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:31:28.370 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:31:28.370 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:31:28.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:31:28.370 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:31:28.370 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:31:28.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 5 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:31:28.371 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: osdmap e81: 8 total, 8 up, 8 in 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: failed to encode map e81 with expected crc 2024-03-30T13:31:28.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: failed to encode map e81 with expected crc 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: failed to encode map e81 with expected crc 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-30T13:31:28.632 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:28 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3362655046' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:31:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: failed to encode map e80 with expected crc 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: osdmap e81: 8 total, 8 up, 8 in 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: failed to encode map e81 with expected crc 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: failed to encode map e81 with expected crc 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: failed to encode map e81 with expected crc 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:31:28.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:31:28.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:31:28.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:31:28.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-30T13:31:28.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:28 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3362655046' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:31:28.900 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:31:28.901 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "7/23 daemons upgraded", 2024-03-30T13:31:28.901 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:31:28.901 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:31:28.901 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:31:28.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 220175 11.14 MB/sec execute 350 sec latency 859.551 ms 2024-03-30T13:31:29.479 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 191/1677 objects degraded (11.389%), 26 pgs degraded 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 191/1677 objects degraded (11.389%), 26 pgs degraded 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-03-30T13:31:29.480 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-03-30T13:31:29.481 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: pgmap v81: 97 pgs: 26 active+undersized+degraded, 2 active+undersized, 69 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 1 op/s; 191/1677 objects degraded (11.389%); 0 B/s, 4 objects/s recovering 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: failed to encode map e81 with expected crc 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: failed to encode map e81 with expected crc 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: failed to encode map e81 with expected crc 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: failed to encode map e81 with expected crc 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: pgmap v83: 97 pgs: 26 active+undersized+degraded, 2 active+undersized, 69 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s wr, 1 op/s; 191/1677 objects degraded (11.389%); 0 B/s, 5 objects/s recovering 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: osdmap e82: 8 total, 8 up, 8 in 2024-03-30T13:31:29.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: failed to encode map e82 with expected crc 2024-03-30T13:31:29.647 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:29 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/37791570' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:31:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:31:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: pgmap v81: 97 pgs: 26 active+undersized+degraded, 2 active+undersized, 69 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 1 op/s; 191/1677 objects degraded (11.389%); 0 B/s, 4 objects/s recovering 2024-03-30T13:31:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: failed to encode map e81 with expected crc 2024-03-30T13:31:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: failed to encode map e81 with expected crc 2024-03-30T13:31:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: failed to encode map e81 with expected crc 2024-03-30T13:31:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: failed to encode map e81 with expected crc 2024-03-30T13:31:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: pgmap v83: 97 pgs: 26 active+undersized+degraded, 2 active+undersized, 69 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s wr, 1 op/s; 191/1677 objects degraded (11.389%); 0 B/s, 5 objects/s recovering 2024-03-30T13:31:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-30T13:31:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-03-30T13:31:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: osdmap e82: 8 total, 8 up, 8 in 2024-03-30T13:31:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: failed to encode map e82 with expected crc 2024-03-30T13:31:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:29 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/37791570' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:31:29.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 220489 11.13 MB/sec execute 351 sec latency 479.203 ms 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: osdmap e83: 8 total, 8 up, 8 in 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: failed to encode map e83 with expected crc 2024-03-30T13:31:30.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:30 smithi159 ceph-mon[67002]: failed to encode map e83 with expected crc 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: failed to encode map e82 with expected crc 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: osdmap e83: 8 total, 8 up, 8 in 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: failed to encode map e83 with expected crc 2024-03-30T13:31:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:30 smithi187 ceph-mon[53604]: failed to encode map e83 with expected crc 2024-03-30T13:31:30.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 220846 11.14 MB/sec execute 352 sec latency 570.447 ms 2024-03-30T13:31:31.705 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: pgmap v86: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.5 MiB/s wr, 679 op/s; 136/1692 objects degraded (8.038%); 3.7 MiB/s, 37 objects/s recovering 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: osdmap e84: 8 total, 8 up, 8 in 2024-03-30T13:31:31.706 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:31 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: failed to encode map e83 with expected crc 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: pgmap v86: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.5 MiB/s wr, 679 op/s; 136/1692 objects degraded (8.038%); 3.7 MiB/s, 37 objects/s recovering 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: osdmap e84: 8 total, 8 up, 8 in 2024-03-30T13:31:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:31 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:31.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 221254 11.15 MB/sec execute 353 sec latency 639.100 ms 2024-03-30T13:31:32.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:32 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:32 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:32 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:32 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:32 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:32 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:32 smithi159 ceph-mon[67002]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:32 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 136/1692 objects degraded (8.038%), 7 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:32 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:32 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:32 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:32 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:32 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:32 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:32 smithi187 ceph-mon[53604]: failed to encode map e84 with expected crc 2024-03-30T13:31:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:32 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 136/1692 objects degraded (8.038%), 7 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:32.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 221647 11.15 MB/sec execute 354 sec latency 50.389 ms 2024-03-30T13:31:33.737 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:33 smithi159 ceph-mon[67002]: pgmap v88: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 136/1692 objects degraded (8.038%); 2.9 MiB/s, 24 objects/s recovering 2024-03-30T13:31:33.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:33 smithi187 ceph-mon[53604]: pgmap v88: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 136/1692 objects degraded (8.038%); 2.9 MiB/s, 24 objects/s recovering 2024-03-30T13:31:33.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 221921 11.13 MB/sec execute 355 sec latency 57.081 ms 2024-03-30T13:31:34.762 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:34 smithi159 ceph-mon[67002]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 136/1692 objects degraded (8.038%), 7 pgs degraded) 2024-03-30T13:31:34.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 222255 11.13 MB/sec execute 356 sec latency 39.121 ms 2024-03-30T13:31:34.946 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:34 smithi187 ceph-mon[53604]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 136/1692 objects degraded (8.038%), 7 pgs degraded) 2024-03-30T13:31:35.804 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:35 smithi159 ceph-mon[67002]: pgmap v89: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 803 op/s; 2.0 MiB/s, 41 objects/s recovering 2024-03-30T13:31:35.916 INFO:teuthology.orchestra.run.smithi159.stdout: 5 222544 11.11 MB/sec execute 357 sec latency 79.059 ms 2024-03-30T13:31:36.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:35 smithi187 ceph-mon[53604]: pgmap v89: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 803 op/s; 2.0 MiB/s, 41 objects/s recovering 2024-03-30T13:31:36.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 222913 11.12 MB/sec execute 358 sec latency 48.843 ms 2024-03-30T13:31:37.592 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:37 smithi159 ceph-mon[67002]: pgmap v90: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 364 op/s; 0 B/s, 20 objects/s recovering 2024-03-30T13:31:37.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 223255 11.12 MB/sec execute 359 sec latency 74.748 ms 2024-03-30T13:31:38.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:37 smithi187 ceph-mon[53604]: pgmap v90: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 364 op/s; 0 B/s, 20 objects/s recovering 2024-03-30T13:31:38.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 223549 11.09 MB/sec execute 360 sec latency 115.361 ms 2024-03-30T13:31:39.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:39 smithi159 ceph-mon[67002]: pgmap v91: 97 pgs: 97 active+clean; 350 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 339 op/s; 0 B/s, 18 objects/s recovering 2024-03-30T13:31:39.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 223800 11.07 MB/sec execute 361 sec latency 58.834 ms 2024-03-30T13:31:40.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:39 smithi187 ceph-mon[53604]: pgmap v91: 97 pgs: 97 active+clean; 350 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 339 op/s; 0 B/s, 18 objects/s recovering 2024-03-30T13:31:40.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 224009 11.06 MB/sec execute 362 sec latency 55.640 ms 2024-03-30T13:31:41.732 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:41 smithi159 ceph-mon[67002]: pgmap v92: 97 pgs: 97 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 514 op/s; 0 B/s, 14 objects/s recovering 2024-03-30T13:31:41.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 224275 11.06 MB/sec execute 363 sec latency 46.004 ms 2024-03-30T13:31:42.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:41 smithi187 ceph-mon[53604]: pgmap v92: 97 pgs: 97 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 514 op/s; 0 B/s, 14 objects/s recovering 2024-03-30T13:31:42.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 224499 11.04 MB/sec execute 364 sec latency 61.100 ms 2024-03-30T13:31:43.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:43 smithi159 ceph-mon[67002]: pgmap v93: 97 pgs: 97 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 459 op/s; 0 B/s, 13 objects/s recovering 2024-03-30T13:31:43.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:43.590 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:31:43.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:43.591 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-30T13:31:43.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:43 smithi187 ceph-mon[53604]: pgmap v93: 97 pgs: 97 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 459 op/s; 0 B/s, 13 objects/s recovering 2024-03-30T13:31:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:31:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-30T13:31:43.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 224986 11.07 MB/sec execute 365 sec latency 36.381 ms 2024-03-30T13:31:44.610 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:44 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-30T13:31:44.610 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:44 smithi159 ceph-mon[67002]: Upgrade: osd.1 is safe to restart 2024-03-30T13:31:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:44 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-30T13:31:44.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:44 smithi187 ceph-mon[53604]: Upgrade: osd.1 is safe to restart 2024-03-30T13:31:44.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 225368 11.05 MB/sec execute 366 sec latency 43.262 ms 2024-03-30T13:31:45.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:45 smithi159 ceph-mon[67002]: pgmap v94: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.1 MiB/s wr, 594 op/s; 0 B/s, 12 objects/s recovering 2024-03-30T13:31:45.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:45 smithi187 ceph-mon[53604]: pgmap v94: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.1 MiB/s wr, 594 op/s; 0 B/s, 12 objects/s recovering 2024-03-30T13:31:45.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 225826 11.05 MB/sec execute 367 sec latency 36.278 ms 2024-03-30T13:31:46.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 226179 11.05 MB/sec execute 368 sec latency 34.134 ms 2024-03-30T13:31:47.153 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:47 smithi159 ceph-mon[67002]: Upgrade: Updating osd.1 2024-03-30T13:31:47.153 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:47.154 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-30T13:31:47.154 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:31:47.154 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:47 smithi159 ceph-mon[67002]: Deploying daemon osd.1 on smithi159 2024-03-30T13:31:47.154 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:47 smithi159 ceph-mon[67002]: pgmap v95: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 375 op/s 2024-03-30T13:31:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:47 smithi187 ceph-mon[53604]: Upgrade: Updating osd.1 2024-03-30T13:31:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-30T13:31:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:31:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:47 smithi187 ceph-mon[53604]: Deploying daemon osd.1 on smithi159 2024-03-30T13:31:47.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:47 smithi187 ceph-mon[53604]: pgmap v95: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 375 op/s 2024-03-30T13:31:47.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 226655 11.07 MB/sec execute 369 sec latency 81.467 ms 2024-03-30T13:31:48.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 227023 11.05 MB/sec execute 370 sec latency 72.817 ms 2024-03-30T13:31:49.444 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:49 smithi159 ceph-mon[67002]: pgmap v96: 97 pgs: 97 active+clean; 346 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-03-30T13:31:49.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:49 smithi187 ceph-mon[53604]: pgmap v96: 97 pgs: 97 active+clean; 346 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-03-30T13:31:49.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 227342 11.03 MB/sec execute 371 sec latency 47.963 ms 2024-03-30T13:31:50.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 227616 11.03 MB/sec execute 372 sec latency 37.323 ms 2024-03-30T13:31:51.482 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:51 smithi159 ceph-mon[67002]: pgmap v97: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 614 op/s 2024-03-30T13:31:51.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:51 smithi187 ceph-mon[53604]: pgmap v97: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 614 op/s 2024-03-30T13:31:51.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 227919 11.03 MB/sec execute 373 sec latency 40.252 ms 2024-03-30T13:31:52.512 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:52 smithi159 systemd[1]: Stopping Ceph osd.1 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:31:52.764 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:52 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[37995]: 2024-03-30T13:31:52.653+0000 7f6d39ca9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:31:52.764 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:52 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[37995]: 2024-03-30T13:31:52.653+0000 7f6d39ca9700 -1 osd.1 84 *** Got signal Terminated *** 2024-03-30T13:31:52.765 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:52 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[37995]: 2024-03-30T13:31:52.653+0000 7f6d39ca9700 -1 osd.1 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:31:52.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 228231 11.03 MB/sec execute 374 sec latency 65.472 ms 2024-03-30T13:31:53.292 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:53 smithi159 ceph-mon[67002]: pgmap v98: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 418 op/s 2024-03-30T13:31:53.292 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:53 smithi159 ceph-mon[67002]: osd.1 marked itself down and dead 2024-03-30T13:31:53.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:53 smithi187 ceph-mon[53604]: pgmap v98: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 418 op/s 2024-03-30T13:31:53.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:53 smithi187 ceph-mon[53604]: osd.1 marked itself down and dead 2024-03-30T13:31:53.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 228394 11.02 MB/sec execute 375 sec latency 670.010 ms 2024-03-30T13:31:54.239 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:53 smithi159 podman[75574]: 2024-03-30 13:31:53.900167015 +0000 UTC m=+1.379810140 container died 1213b805bf8ad233702620de05167f04b3f21b03c85e8a98306978b6b583be65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_CLEAN=True, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: osdmap e85: 8 total, 7 up, 8 in 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.496 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:54 smithi159 ceph-mon[67002]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:31:54.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: osdmap e85: 8 total, 7 up, 8 in 2024-03-30T13:31:54.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:54 smithi187 ceph-mon[53604]: failed to encode map e85 with expected crc 2024-03-30T13:31:54.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 228521 11.00 MB/sec execute 376 sec latency 1034.833 ms 2024-03-30T13:31:55.035 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:54 smithi159 podman[75574]: 2024-03-30 13:31:54.910094571 +0000 UTC m=+2.389737706 container cleanup 1213b805bf8ad233702620de05167f04b3f21b03c85e8a98306978b6b583be65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, distribution-scope=public, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, release=754, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vcs-type=git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=) 2024-03-30T13:31:55.036 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:54 smithi159 bash[75574]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1 2024-03-30T13:31:55.301 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: pgmap v100: 97 pgs: 23 peering, 74 active+clean; 346 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 529 op/s 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: osdmap e86: 8 total, 7 up, 8 in 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.302 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.303 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.303 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.303 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.303 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:55 smithi159 ceph-mon[67002]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: pgmap v100: 97 pgs: 23 peering, 74 active+clean; 346 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 529 op/s 2024-03-30T13:31:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: osdmap e86: 8 total, 7 up, 8 in 2024-03-30T13:31:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:55 smithi187 ceph-mon[53604]: failed to encode map e86 with expected crc 2024-03-30T13:31:55.553 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:55 smithi159 podman[75590]: 2024-03-30 13:31:55.344233795 +0000 UTC m=+1.442700559 container remove 1213b805bf8ad233702620de05167f04b3f21b03c85e8a98306978b6b583be65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, version=8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, name=centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base centos centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-03-30T13:31:55.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 228764 10.98 MB/sec execute 377 sec latency 56.263 ms 2024-03-30T13:31:56.075 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:55 smithi159 podman[75678]: 2024-03-30 13:31:55.747834289 +0000 UTC m=+0.023058537 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:31:56.076 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:55 smithi159 podman[75678]: 2024-03-30 13:31:55.928387122 +0000 UTC m=+0.203611357 container create f202734bb02b41d1d6356da699b10022c5956b4d86f5e1aa9f3f477cad6f5c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:31:56.345 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:56 smithi159 podman[75678]: 2024-03-30 13:31:56.243126007 +0000 UTC m=+0.518350241 container init f202734bb02b41d1d6356da699b10022c5956b4d86f5e1aa9f3f477cad6f5c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:31:56.345 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:56 smithi159 podman[75678]: 2024-03-30 13:31:56.248349929 +0000 UTC m=+0.523574166 container start f202734bb02b41d1d6356da699b10022c5956b4d86f5e1aa9f3f477cad6f5c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:31:56.596 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:56 smithi159 podman[75678]: 2024-03-30 13:31:56.365692561 +0000 UTC m=+0.640916794 container attach f202734bb02b41d1d6356da699b10022c5956b4d86f5e1aa9f3f477cad6f5c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD) 2024-03-30T13:31:56.848 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:56 smithi159 podman[75678]: 2024-03-30 13:31:56.632578682 +0000 UTC m=+0.907802909 container died f202734bb02b41d1d6356da699b10022c5956b4d86f5e1aa9f3f477cad6f5c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:31:56.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 228976 10.96 MB/sec execute 378 sec latency 55.075 ms 2024-03-30T13:31:57.354 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:57 smithi159 ceph-mon[67002]: pgmap v102: 97 pgs: 13 stale+active+clean, 23 peering, 61 active+clean; 346 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.2 MiB/s wr, 640 op/s 2024-03-30T13:31:57.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:57 smithi187 ceph-mon[53604]: pgmap v102: 97 pgs: 13 stale+active+clean, 23 peering, 61 active+clean; 346 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.2 MiB/s wr, 640 op/s 2024-03-30T13:31:57.615 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:57 smithi159 podman[75703]: 2024-03-30 13:31:57.608651225 +0000 UTC m=+0.971212759 container remove f202734bb02b41d1d6356da699b10022c5956b4d86f5e1aa9f3f477cad6f5c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-03-30T13:31:57.872 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:57 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.1.service: Deactivated successfully. 2024-03-30T13:31:57.873 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:57 smithi159 systemd[1]: Stopped Ceph osd.1 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:31:57.873 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:57 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.1.service: Consumed 4min 26.768s CPU time. 2024-03-30T13:31:57.917 INFO:teuthology.orchestra.run.smithi159.stdout: 5 229257 10.94 MB/sec execute 379 sec latency 41.756 ms 2024-03-30T13:31:58.124 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:57 smithi159 systemd[1]: Starting Ceph osd.1 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:31:58.391 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:58 smithi159 podman[75799]: 2024-03-30 13:31:58.115401847 +0000 UTC m=+0.022369928 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:31:58.644 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:58 smithi159 podman[75799]: 2024-03-30 13:31:58.39792562 +0000 UTC m=+0.304893700 container create 64571921982b5c0ab531de02a180489cd5c5d69fb03f2438929b5ca38a29401a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:31:58.644 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:58.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:31:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:31:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:31:58.914 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:58 smithi159 podman[75799]: 2024-03-30 13:31:58.814582376 +0000 UTC m=+0.721550457 container init 64571921982b5c0ab531de02a180489cd5c5d69fb03f2438929b5ca38a29401a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-30T13:31:58.914 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:58 smithi159 podman[75799]: 2024-03-30 13:31:58.823273386 +0000 UTC m=+0.730241471 container start 64571921982b5c0ab531de02a180489cd5c5d69fb03f2438929b5ca38a29401a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-03-30T13:31:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 229555 10.94 MB/sec execute 380 sec latency 60.347 ms 2024-03-30T13:31:59.183 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:58 smithi159 podman[75799]: 2024-03-30 13:31:58.939776092 +0000 UTC m=+0.846744183 container attach 64571921982b5c0ab531de02a180489cd5c5d69fb03f2438929b5ca38a29401a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS) 2024-03-30T13:31:59.434 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate[75813]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate[75813]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate[75813]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate[75813]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate[75813]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate[75813]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate[75813]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 bash[75799]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 bash[75799]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 bash[75799]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-30T13:31:59.435 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 bash[75799]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-30T13:31:59.436 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 bash[75799]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-30T13:31:59.436 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 bash[75799]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-30T13:31:59.436 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 bash[75799]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-30T13:31:59.664 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:31:59 smithi159 podman[75799]: 2024-03-30 13:31:59.469844551 +0000 UTC m=+1.376812632 container died 64571921982b5c0ab531de02a180489cd5c5d69fb03f2438929b5ca38a29401a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc) 2024-03-30T13:31:59.665 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-mon[67002]: pgmap v103: 97 pgs: 12 stale+active+clean, 2 active+undersized+degraded, 23 peering, 2 active+undersized, 58 active+clean; 346 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.2 MiB/s wr, 291 op/s; 23/1623 objects degraded (1.417%) 2024-03-30T13:31:59.665 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:31:59 smithi159 ceph-mon[67002]: Health check failed: Degraded data redundancy: 23/1623 objects degraded (1.417%), 2 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:59.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:59 smithi187 ceph-mon[53604]: pgmap v103: 97 pgs: 12 stale+active+clean, 2 active+undersized+degraded, 23 peering, 2 active+undersized, 58 active+clean; 346 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.2 MiB/s wr, 291 op/s; 23/1623 objects degraded (1.417%) 2024-03-30T13:31:59.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:31:59 smithi187 ceph-mon[53604]: Health check failed: Degraded data redundancy: 23/1623 objects degraded (1.417%), 2 pgs degraded (PG_DEGRADED) 2024-03-30T13:31:59.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 229839 10.93 MB/sec execute 381 sec latency 49.483 ms 2024-03-30T13:31:59.992 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:32:00.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 230175 10.94 MB/sec execute 382 sec latency 69.892 ms 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (5m) 39s ago 14m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (88s) 39s ago 14m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (82s) 76s ago 14m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 39s ago 14m - - 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (8m) 39s ago 8m 38.4M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (8m) 76s ago 8m 16.8M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (4m) 39s ago 15m 502M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (3m) 76s ago 13m 449M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (2m) 39s ago 16m 43.8M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:32:00.951 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (2m) 76s ago 13m 34.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (8m) 39s ago 8m 282M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (8m) 76s ago 8m 86.1M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (4m) 39s ago 14m 10.4M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (4m) 76s ago 13m 18.3M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (49s) 39s ago 13m 12.7M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (12m) 39s ago 12m 376M 1936M 17.2.0 e1d6a67b021e 1213b805bf8a 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (12m) 39s ago 12m 562M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (11m) 39s ago 11m 384M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (11m) 76s ago 11m 595M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (11m) 76s ago 11m 465M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (10m) 76s ago 10m 378M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (10m) 76s ago 10m 538M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:32:00.952 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (3m) 39s ago 13m 62.9M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:32:01.208 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 podman[75950]: 2024-03-30 13:32:01.049091797 +0000 UTC m=+1.573465080 container remove 64571921982b5c0ab531de02a180489cd5c5d69fb03f2438929b5ca38a29401a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-30T13:32:01.460 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 podman[76093]: 2024-03-30 13:32:01.236188152 +0000 UTC m=+0.036713269 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:32:01.460 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 podman[76093]: 2024-03-30 13:32:01.399968669 +0000 UTC m=+0.200493757 container create 2d27b1045a3df567528c4fc35314db0e9df9357dc4918f34e1367fcb03aa334d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-30T13:32:01.603 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:32:01.603 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:32:01.603 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:32:01.603 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:32:01.603 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 1 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 5 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:32:01.604 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:32:01.717 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:01 smithi159 ceph-mon[67002]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:01.718 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:01 smithi159 ceph-mon[67002]: pgmap v104: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 491 op/s; 195/1692 objects degraded (11.525%) 2024-03-30T13:32:01.718 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:01 smithi159 ceph-mon[67002]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:01.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:01 smithi187 ceph-mon[53604]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:01.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:01 smithi187 ceph-mon[53604]: pgmap v104: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 491 op/s; 195/1692 objects degraded (11.525%) 2024-03-30T13:32:01.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:01 smithi187 ceph-mon[53604]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:01.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 230435 10.92 MB/sec execute 383 sec latency 77.707 ms 2024-03-30T13:32:01.980 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 podman[76093]: 2024-03-30 13:32:01.752226125 +0000 UTC m=+0.552751239 container init 2d27b1045a3df567528c4fc35314db0e9df9357dc4918f34e1367fcb03aa334d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-30T13:32:01.980 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 podman[76093]: 2024-03-30 13:32:01.761100694 +0000 UTC m=+0.561625784 container start 2d27b1045a3df567528c4fc35314db0e9df9357dc4918f34e1367fcb03aa334d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-4e50bbc, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux ) 2024-03-30T13:32:01.980 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 ceph-osd[76115]: -- 172.21.15.159:0/683813461 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 ==== 357+0+0 (secure 0 0 0) 0x562a8bc31ba0 con 0x562a8bc02c00 2024-03-30T13:32:01.981 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 ceph-osd[76115]: -- 172.21.15.159:0/683813461 <== mon.0 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562a8c9e81e0 con 0x562a8bc02c00 2024-03-30T13:32:01.981 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 bash[76093]: 2d27b1045a3df567528c4fc35314db0e9df9357dc4918f34e1367fcb03aa334d 2024-03-30T13:32:01.981 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:01 smithi159 systemd[1]: Started Ceph osd.1 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:32:02.170 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:32:02.170 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:32:02.170 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:32:02.170 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:32:02.170 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:32:02.171 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:32:02.171 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:32:02.171 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:32:02.171 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:32:02.171 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "7/23 daemons upgraded", 2024-03-30T13:32:02.171 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:32:02.171 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:32:02.171 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:32:02.248 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:02 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:32:02.099+0000 7f1884d087c0 -1 Falling back to public interface 2024-03-30T13:32:02.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:02 smithi159 ceph-mon[67002]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:02.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:02 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3752410151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:02.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:02.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:02.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:32:02.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:02 smithi187 ceph-mon[53604]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:02 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3752410151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:32:02.833 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 195/1692 objects degraded (11.525%), 24 pgs degraded 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: osd.1 (root=default,host=smithi159) is down 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 195/1692 objects degraded (11.525%), 24 pgs degraded 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-03-30T13:32:02.834 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-03-30T13:32:02.835 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-03-30T13:32:02.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 230677 10.90 MB/sec execute 384 sec latency 45.019 ms 2024-03-30T13:32:03.535 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:03 smithi159 ceph-mon[67002]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:03.535 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:03 smithi159 ceph-mon[67002]: pgmap v105: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 417 op/s; 195/1692 objects degraded (11.525%) 2024-03-30T13:32:03.536 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:03 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1996406465' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:32:03.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:03 smithi187 ceph-mon[53604]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:03.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:03 smithi187 ceph-mon[53604]: pgmap v105: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 417 op/s; 195/1692 objects degraded (11.525%) 2024-03-30T13:32:03.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:03 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1996406465' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:32:03.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 230868 10.88 MB/sec execute 385 sec latency 57.186 ms 2024-03-30T13:32:04.601 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:04 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 187/1638 objects degraded (11.416%), 24 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:04.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 231117 10.87 MB/sec execute 386 sec latency 50.161 ms 2024-03-30T13:32:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:04 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 187/1638 objects degraded (11.416%), 24 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:05.620 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:05 smithi159 ceph-mon[67002]: pgmap v106: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 364 op/s; 187/1638 objects degraded (11.416%) 2024-03-30T13:32:05.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 231579 10.88 MB/sec execute 387 sec latency 48.944 ms 2024-03-30T13:32:06.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:05 smithi187 ceph-mon[53604]: pgmap v106: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 364 op/s; 187/1638 objects degraded (11.416%) 2024-03-30T13:32:06.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 232064 10.91 MB/sec execute 388 sec latency 48.661 ms 2024-03-30T13:32:07.654 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:07 smithi159 ceph-mon[67002]: pgmap v107: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.9 MiB/s wr, 305 op/s; 187/1638 objects degraded (11.416%) 2024-03-30T13:32:07.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 232472 10.89 MB/sec execute 389 sec latency 33.786 ms 2024-03-30T13:32:08.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:07 smithi187 ceph-mon[53604]: pgmap v107: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.9 MiB/s wr, 305 op/s; 187/1638 objects degraded (11.416%) 2024-03-30T13:32:08.917 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:08 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:32:08.696+0000 7f1884d087c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-03-30T13:32:08.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 232725 10.87 MB/sec execute 390 sec latency 58.746 ms 2024-03-30T13:32:09.678 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:09 smithi159 ceph-mon[67002]: pgmap v108: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 318 op/s; 188/1644 objects degraded (11.436%) 2024-03-30T13:32:09.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 233047 10.87 MB/sec execute 391 sec latency 46.810 ms 2024-03-30T13:32:10.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:09 smithi187 ceph-mon[53604]: pgmap v108: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 318 op/s; 188/1644 objects degraded (11.436%) 2024-03-30T13:32:10.201 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:10 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:32:10.196+0000 7f1884d087c0 -1 osd.1 84 log_to_monitors true 2024-03-30T13:32:10.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:10 smithi159 ceph-mon[67002]: from='osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-30T13:32:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 233308 10.86 MB/sec execute 392 sec latency 79.222 ms 2024-03-30T13:32:11.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:10 smithi187 ceph-mon[53604]: from='osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-30T13:32:11.218 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:32:11.133+0000 7f1872158700 -1 osd.1 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:32:11.783 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: pgmap v109: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 350 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 501 op/s; 193/1710 objects degraded (11.287%) 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 193/1710 objects degraded (11.287%), 24 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: from='osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: failed to encode map e87 with expected crc 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: osdmap e87: 8 total, 7 up, 8 in 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: from='osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: failed to encode map e87 with expected crc 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: failed to encode map e87 with expected crc 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:11.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:11 smithi159 ceph-mon[67002]: from='osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926]' entity='osd.1' 2024-03-30T13:32:11.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 233660 10.88 MB/sec execute 393 sec latency 72.509 ms 2024-03-30T13:32:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: pgmap v109: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 350 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 501 op/s; 193/1710 objects degraded (11.287%) 2024-03-30T13:32:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 193/1710 objects degraded (11.287%), 24 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: from='osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-30T13:32:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: failed to encode map e87 with expected crc 2024-03-30T13:32:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: osdmap e87: 8 total, 7 up, 8 in 2024-03-30T13:32:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: from='osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:32:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: failed to encode map e87 with expected crc 2024-03-30T13:32:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: failed to encode map e87 with expected crc 2024-03-30T13:32:12.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:12.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:12.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:11 smithi187 ceph-mon[53604]: from='osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926]' entity='osd.1' 2024-03-30T13:32:12.855 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:32:12.855 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: failed to encode map e88 with expected crc 2024-03-30T13:32:12.855 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926] boot 2024-03-30T13:32:12.855 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: osdmap e88: 8 total, 8 up, 8 in 2024-03-30T13:32:12.855 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:32:12.855 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: failed to encode map e88 with expected crc 2024-03-30T13:32:12.856 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: failed to encode map e88 with expected crc 2024-03-30T13:32:12.856 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: failed to encode map e88 with expected crc 2024-03-30T13:32:12.856 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: failed to encode map e88 with expected crc 2024-03-30T13:32:12.856 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:12.856 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:12.856 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:12.856 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:32:12.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 233732 10.85 MB/sec execute 394 sec latency 747.184 ms 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: failed to encode map e88 with expected crc 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: osd.1 [v2:172.21.15.159:6810/19316926,v1:172.21.15.159:6811/19316926] boot 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: osdmap e88: 8 total, 8 up, 8 in 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: failed to encode map e88 with expected crc 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: failed to encode map e88 with expected crc 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: failed to encode map e88 with expected crc 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: failed to encode map e88 with expected crc 2024-03-30T13:32:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:13.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:32:13.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e88 with expected crc 2024-03-30T13:32:13.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: pgmap v112: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.3 MiB/s wr, 306 op/s; 193/1710 objects degraded (11.287%) 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: osdmap e89: 8 total, 8 up, 8 in 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:13.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:13 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:13.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 233874 10.83 MB/sec execute 395 sec latency 1706.921 ms 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e88 with expected crc 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: pgmap v112: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.3 MiB/s wr, 306 op/s; 193/1710 objects degraded (11.287%) 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: osdmap e89: 8 total, 8 up, 8 in 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:13 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:14 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.663 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:14 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.663 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:14 smithi159 ceph-mon[67002]: failed to encode map e89 with expected crc 2024-03-30T13:32:14.663 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:14 smithi159 ceph-mon[67002]: failed to encode map e90 with expected crc 2024-03-30T13:32:14.663 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:14 smithi159 ceph-mon[67002]: osdmap e90: 8 total, 8 up, 8 in 2024-03-30T13:32:14.663 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:14 smithi159 ceph-mon[67002]: failed to encode map e90 with expected crc 2024-03-30T13:32:14.663 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:14 smithi159 ceph-mon[67002]: failed to encode map e90 with expected crc 2024-03-30T13:32:14.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 234140 10.81 MB/sec execute 396 sec latency 1985.493 ms 2024-03-30T13:32:15.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:14 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:15.031 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:14 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:15.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:14 smithi187 ceph-mon[53604]: failed to encode map e89 with expected crc 2024-03-30T13:32:15.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:14 smithi187 ceph-mon[53604]: failed to encode map e90 with expected crc 2024-03-30T13:32:15.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:14 smithi187 ceph-mon[53604]: osdmap e90: 8 total, 8 up, 8 in 2024-03-30T13:32:15.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:14 smithi187 ceph-mon[53604]: failed to encode map e90 with expected crc 2024-03-30T13:32:15.032 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:14 smithi187 ceph-mon[53604]: failed to encode map e90 with expected crc 2024-03-30T13:32:15.680 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e90 with expected crc 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e90 with expected crc 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: pgmap v115: 97 pgs: 75 active+clean, 4 active+undersized, 1 active+recovering+degraded, 3 active+recovery_wait+degraded, 6 active+undersized+degraded, 3 peering, 5 remapped+peering; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 3.6 MiB/s wr, 256 op/s; 88/1593 objects degraded (5.524%); 0 B/s, 5 objects/s recovering 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: osdmap e91: 8 total, 8 up, 8 in 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:15.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:15 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:15.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 234326 10.78 MB/sec execute 397 sec latency 693.338 ms 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e90 with expected crc 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e90 with expected crc 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: pgmap v115: 97 pgs: 75 active+clean, 4 active+undersized, 1 active+recovering+degraded, 3 active+recovery_wait+degraded, 6 active+undersized+degraded, 3 peering, 5 remapped+peering; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 3.6 MiB/s wr, 256 op/s; 88/1593 objects degraded (5.524%); 0 B/s, 5 objects/s recovering 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: osdmap e91: 8 total, 8 up, 8 in 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:15 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.726 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e91 with expected crc 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: osdmap e92: 8 total, 8 up, 8 in 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:16.727 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:16 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:16.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 234610 10.78 MB/sec execute 398 sec latency 957.886 ms 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e91 with expected crc 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: osdmap e92: 8 total, 8 up, 8 in 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:16 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.746 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.746 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.746 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.746 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: failed to encode map e92 with expected crc 2024-03-30T13:32:17.747 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: pgmap v118: 97 pgs: 84 active+clean, 1 active+recovering+degraded, 3 active+recovery_wait+degraded, 6 peering, 3 remapped+peering; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 522 op/s; 54/1596 objects degraded (3.383%); 22 B/s, 9 objects/s recovering 2024-03-30T13:32:17.747 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 88/1593 objects degraded (5.524%), 10 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:17.747 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: osdmap e93: 8 total, 8 up, 8 in 2024-03-30T13:32:17.747 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: failed to encode map e93 with expected crc 2024-03-30T13:32:17.747 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: failed to encode map e93 with expected crc 2024-03-30T13:32:17.747 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:17 smithi159 ceph-mon[67002]: failed to encode map e93 with expected crc 2024-03-30T13:32:17.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 234948 10.77 MB/sec execute 399 sec latency 995.129 ms 2024-03-30T13:32:18.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: failed to encode map e92 with expected crc 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: pgmap v118: 97 pgs: 84 active+clean, 1 active+recovering+degraded, 3 active+recovery_wait+degraded, 6 peering, 3 remapped+peering; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 522 op/s; 54/1596 objects degraded (3.383%); 22 B/s, 9 objects/s recovering 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 88/1593 objects degraded (5.524%), 10 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: osdmap e93: 8 total, 8 up, 8 in 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: failed to encode map e93 with expected crc 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: failed to encode map e93 with expected crc 2024-03-30T13:32:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:17 smithi187 ceph-mon[53604]: failed to encode map e93 with expected crc 2024-03-30T13:32:18.769 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: failed to encode map e93 with expected crc 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: failed to encode map e93 with expected crc 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: failed to encode map e93 with expected crc 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:32:18.770 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:18.771 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:18.771 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:18.771 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-30T13:32:18.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 235325 10.78 MB/sec execute 400 sec latency 43.821 ms 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: failed to encode map e93 with expected crc 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: failed to encode map e93 with expected crc 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: failed to encode map e93 with expected crc 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:32:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:19.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-30T13:32:19.791 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:19 smithi159 ceph-mon[67002]: pgmap v120: 97 pgs: 87 active+clean, 1 active+recovering+degraded, 3 active+recovery_wait+degraded, 3 peering, 3 remapped+peering; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.9 MiB/s wr, 288 op/s; 54/1581 objects degraded (3.416%); 107 KiB/s, 10 objects/s recovering 2024-03-30T13:32:19.792 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:19 smithi159 ceph-mon[67002]: pgmap v121: 97 pgs: 87 active+clean, 1 active+recovering+degraded, 3 active+recovery_wait+degraded, 3 peering, 3 remapped+peering; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.9 MiB/s wr, 286 op/s; 54/1581 objects degraded (3.416%); 106 KiB/s, 10 objects/s recovering 2024-03-30T13:32:19.792 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:19 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-30T13:32:19.792 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:19 smithi159 ceph-mon[67002]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-30T13:32:19.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 235762 10.79 MB/sec execute 401 sec latency 40.560 ms 2024-03-30T13:32:20.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:19 smithi187 ceph-mon[53604]: pgmap v120: 97 pgs: 87 active+clean, 1 active+recovering+degraded, 3 active+recovery_wait+degraded, 3 peering, 3 remapped+peering; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.9 MiB/s wr, 288 op/s; 54/1581 objects degraded (3.416%); 107 KiB/s, 10 objects/s recovering 2024-03-30T13:32:20.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:19 smithi187 ceph-mon[53604]: pgmap v121: 97 pgs: 87 active+clean, 1 active+recovering+degraded, 3 active+recovery_wait+degraded, 3 peering, 3 remapped+peering; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.9 MiB/s wr, 286 op/s; 54/1581 objects degraded (3.416%); 106 KiB/s, 10 objects/s recovering 2024-03-30T13:32:20.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:19 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-30T13:32:20.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:19 smithi187 ceph-mon[53604]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-30T13:32:20.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:20 smithi159 ceph-mon[67002]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/1581 objects degraded (3.416%), 4 pgs degraded) 2024-03-30T13:32:20.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 236192 10.79 MB/sec execute 402 sec latency 42.729 ms 2024-03-30T13:32:21.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:20 smithi187 ceph-mon[53604]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/1581 objects degraded (3.416%), 4 pgs degraded) 2024-03-30T13:32:21.901 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:21 smithi159 ceph-mon[67002]: pgmap v122: 97 pgs: 93 active+clean, 3 peering, 1 remapped+peering; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 577 op/s; 84 KiB/s, 29 objects/s recovering 2024-03-30T13:32:21.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 236553 10.79 MB/sec execute 403 sec latency 29.609 ms 2024-03-30T13:32:22.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:21 smithi187 ceph-mon[53604]: pgmap v122: 97 pgs: 93 active+clean, 3 peering, 1 remapped+peering; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 577 op/s; 84 KiB/s, 29 objects/s recovering 2024-03-30T13:32:22.914 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:22 smithi159 ceph-mon[67002]: pgmap v123: 97 pgs: 97 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.2 MiB/s wr, 332 op/s; 1.4 MiB/s, 36 keys/s, 29 objects/s recovering 2024-03-30T13:32:22.918 INFO:teuthology.orchestra.run.smithi159.stdout: 5 236922 10.79 MB/sec execute 404 sec latency 58.034 ms 2024-03-30T13:32:23.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:22 smithi187 ceph-mon[53604]: pgmap v123: 97 pgs: 97 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.2 MiB/s wr, 332 op/s; 1.4 MiB/s, 36 keys/s, 29 objects/s recovering 2024-03-30T13:32:23.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 237273 10.78 MB/sec execute 405 sec latency 66.690 ms 2024-03-30T13:32:24.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 237621 10.77 MB/sec execute 406 sec latency 47.629 ms 2024-03-30T13:32:25.446 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:25 smithi159 ceph-mon[67002]: pgmap v124: 97 pgs: 97 active+clean; 355 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.1 MiB/s wr, 593 op/s; 1.0 MiB/s, 27 keys/s, 22 objects/s recovering 2024-03-30T13:32:25.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:25 smithi187 ceph-mon[53604]: pgmap v124: 97 pgs: 97 active+clean; 355 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.1 MiB/s wr, 593 op/s; 1.0 MiB/s, 27 keys/s, 22 objects/s recovering 2024-03-30T13:32:25.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 237992 10.76 MB/sec execute 407 sec latency 59.169 ms 2024-03-30T13:32:26.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 238435 10.76 MB/sec execute 408 sec latency 41.257 ms 2024-03-30T13:32:27.492 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:27 smithi159 ceph-mon[67002]: pgmap v125: 97 pgs: 97 active+clean; 354 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.6 MiB/s wr, 537 op/s; 946 KiB/s, 24 keys/s, 20 objects/s recovering 2024-03-30T13:32:27.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:27 smithi187 ceph-mon[53604]: pgmap v125: 97 pgs: 97 active+clean; 354 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.6 MiB/s wr, 537 op/s; 946 KiB/s, 24 keys/s, 20 objects/s recovering 2024-03-30T13:32:27.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 238896 10.78 MB/sec execute 409 sec latency 37.876 ms 2024-03-30T13:32:28.509 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:28.509 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:32:28.509 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:28.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:32:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:28.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 239216 10.78 MB/sec execute 410 sec latency 43.775 ms 2024-03-30T13:32:29.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:29 smithi159 ceph-mon[67002]: pgmap v126: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 494 op/s; 813 KiB/s, 22 keys/s, 15 objects/s recovering 2024-03-30T13:32:29.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 239550 10.78 MB/sec execute 411 sec latency 56.232 ms 2024-03-30T13:32:30.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:29 smithi187 ceph-mon[53604]: pgmap v126: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 494 op/s; 813 KiB/s, 22 keys/s, 15 objects/s recovering 2024-03-30T13:32:30.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 239847 10.77 MB/sec execute 412 sec latency 41.202 ms 2024-03-30T13:32:31.568 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:31 smithi159 ceph-mon[67002]: pgmap v127: 97 pgs: 97 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 544 op/s; 682 KiB/s, 18 keys/s, 12 objects/s recovering 2024-03-30T13:32:31.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 240169 10.77 MB/sec execute 413 sec latency 39.662 ms 2024-03-30T13:32:32.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:31 smithi187 ceph-mon[53604]: pgmap v127: 97 pgs: 97 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 544 op/s; 682 KiB/s, 18 keys/s, 12 objects/s recovering 2024-03-30T13:32:32.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 240462 10.76 MB/sec execute 414 sec latency 60.755 ms 2024-03-30T13:32:33.417 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:32:33.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:33 smithi159 ceph-mon[67002]: pgmap v128: 97 pgs: 97 active+clean; 354 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 501 op/s; 682 KiB/s, 18 keys/s, 3 objects/s recovering 2024-03-30T13:32:33.588 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:33 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-30T13:32:33.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 240677 10.75 MB/sec execute 415 sec latency 64.691 ms 2024-03-30T13:32:34.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:33 smithi187 ceph-mon[53604]: pgmap v128: 97 pgs: 97 active+clean; 354 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 501 op/s; 682 KiB/s, 18 keys/s, 3 objects/s recovering 2024-03-30T13:32:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:33 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-30T13:32:34.466 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (5m) 23s ago 15m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (2m) 23s ago 15m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (115s) 110s ago 14m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 23s ago 14m - - 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (9m) 23s ago 9m 38.7M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (9m) 110s ago 9m 16.8M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (4m) 23s ago 16m 504M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (3m) 110s ago 14m 449M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (3m) 23s ago 16m 45.9M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (3m) 110s ago 14m 34.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (9m) 23s ago 9m 282M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:32:34.467 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (9m) 110s ago 9m 86.1M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (5m) 23s ago 14m 13.6M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (5m) 110s ago 14m 18.3M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (82s) 23s ago 13m 268M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (33s) 23s ago 13m 15.3M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (12m) 23s ago 12m 337M 1936M 17.2.0 e1d6a67b021e ca0dcfe54da8 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (12m) 23s ago 12m 419M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (12m) 110s ago 12m 595M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (11m) 110s ago 11m 465M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (11m) 110s ago 11m 378M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (10m) 110s ago 10m 538M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:32:34.468 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (4m) 23s ago 14m 63.0M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:32:34.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:34 smithi159 ceph-mon[67002]: from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:34.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:34 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-30T13:32:34.636 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:34 smithi159 ceph-mon[67002]: Upgrade: osd.2 is safe to restart 2024-03-30T13:32:34.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 240932 10.73 MB/sec execute 416 sec latency 82.226 ms 2024-03-30T13:32:34.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:34 smithi187 ceph-mon[53604]: from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:34.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:34 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-30T13:32:34.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:34 smithi187 ceph-mon[53604]: Upgrade: osd.2 is safe to restart 2024-03-30T13:32:35.100 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:32:35.100 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:32:35.100 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:32:35.100 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:32:35.100 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:32:35.100 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:32:35.100 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:32:35.100 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 6 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:32:35.101 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "8/23 daemons upgraded", 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:32:35.620 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:32:35.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:35 smithi159 ceph-mon[67002]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:35.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:35 smithi159 ceph-mon[67002]: pgmap v129: 97 pgs: 97 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.6 MiB/s wr, 633 op/s 2024-03-30T13:32:35.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:35 smithi159 ceph-mon[67002]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:35.662 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:35 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3128268911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:35.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 241170 10.72 MB/sec execute 417 sec latency 75.326 ms 2024-03-30T13:32:36.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:35 smithi187 ceph-mon[53604]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:35 smithi187 ceph-mon[53604]: pgmap v129: 97 pgs: 97 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.6 MiB/s wr, 633 op/s 2024-03-30T13:32:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:35 smithi187 ceph-mon[53604]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:35 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3128268911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:32:36.187 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:32:36.187 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:32:36.187 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:32:36.686 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:36 smithi159 ceph-mon[67002]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:36.686 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:36.687 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-30T13:32:36.687 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:32:36.687 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:36 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1207642532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:32:36.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 241411 10.70 MB/sec execute 418 sec latency 79.377 ms 2024-03-30T13:32:37.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:36 smithi187 ceph-mon[53604]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:32:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-30T13:32:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:32:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:36 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1207642532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:32:37.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:37 smithi159 ceph-mon[67002]: Upgrade: Updating osd.2 2024-03-30T13:32:37.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:37 smithi159 ceph-mon[67002]: Deploying daemon osd.2 on smithi159 2024-03-30T13:32:37.729 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:37 smithi159 ceph-mon[67002]: pgmap v130: 97 pgs: 97 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 490 op/s 2024-03-30T13:32:37.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 241676 10.69 MB/sec execute 419 sec latency 98.405 ms 2024-03-30T13:32:38.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:37 smithi187 ceph-mon[53604]: Upgrade: Updating osd.2 2024-03-30T13:32:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:37 smithi187 ceph-mon[53604]: Deploying daemon osd.2 on smithi159 2024-03-30T13:32:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:37 smithi187 ceph-mon[53604]: pgmap v130: 97 pgs: 97 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 490 op/s 2024-03-30T13:32:38.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 241889 10.68 MB/sec execute 420 sec latency 68.990 ms 2024-03-30T13:32:39.784 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:39 smithi159 ceph-mon[67002]: pgmap v131: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 490 op/s 2024-03-30T13:32:39.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 242163 10.68 MB/sec execute 421 sec latency 79.193 ms 2024-03-30T13:32:40.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:39 smithi187 ceph-mon[53604]: pgmap v131: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 490 op/s 2024-03-30T13:32:40.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 242463 10.67 MB/sec execute 422 sec latency 51.465 ms 2024-03-30T13:32:41.626 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:41 smithi159 ceph-mon[67002]: pgmap v132: 97 pgs: 97 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.1 MiB/s wr, 561 op/s 2024-03-30T13:32:41.626 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:41 smithi159 systemd[1]: Stopping Ceph osd.2 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:32:41.876 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:41 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[41867]: 2024-03-30T13:32:41.751+0000 7f0e5aef6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:32:41.877 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:41 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[41867]: 2024-03-30T13:32:41.751+0000 7f0e5aef6700 -1 osd.2 93 *** Got signal Terminated *** 2024-03-30T13:32:41.877 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:41 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[41867]: 2024-03-30T13:32:41.751+0000 7f0e5aef6700 -1 osd.2 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:32:41.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 242761 10.67 MB/sec execute 423 sec latency 50.747 ms 2024-03-30T13:32:42.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:41 smithi187 ceph-mon[53604]: pgmap v132: 97 pgs: 97 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.1 MiB/s wr, 561 op/s 2024-03-30T13:32:42.661 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:42 smithi159 ceph-mon[67002]: osd.2 marked itself down and dead 2024-03-30T13:32:42.661 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:32:42.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 243097 10.67 MB/sec execute 424 sec latency 323.015 ms 2024-03-30T13:32:43.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:42 smithi187 ceph-mon[53604]: osd.2 marked itself down and dead 2024-03-30T13:32:43.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:32:43.416 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:43 smithi159 podman[78245]: 2024-03-30 13:32:43.248832953 +0000 UTC m=+1.636007274 container died ca0dcfe54da88164739f15aa67c1468636733bda787912d70069158f02243b27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., architecture=x86_64, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, ceph=True, GIT_CLEAN=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_BRANCH=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, name=centos-stream, io.openshift.expose-services=) 2024-03-30T13:32:43.675 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: pgmap v133: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 447 op/s 2024-03-30T13:32:43.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:32:43.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: failed to encode map e94 with expected crc 2024-03-30T13:32:43.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: osdmap e94: 8 total, 7 up, 8 in 2024-03-30T13:32:43.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: failed to encode map e94 with expected crc 2024-03-30T13:32:43.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: failed to encode map e94 with expected crc 2024-03-30T13:32:43.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: failed to encode map e94 with expected crc 2024-03-30T13:32:43.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: failed to encode map e94 with expected crc 2024-03-30T13:32:43.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:43 smithi159 ceph-mon[67002]: failed to encode map e94 with expected crc 2024-03-30T13:32:43.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 243394 10.66 MB/sec execute 425 sec latency 1002.566 ms 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: pgmap v133: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 447 op/s 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: failed to encode map e94 with expected crc 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: osdmap e94: 8 total, 7 up, 8 in 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: failed to encode map e94 with expected crc 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: failed to encode map e94 with expected crc 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: failed to encode map e94 with expected crc 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: failed to encode map e94 with expected crc 2024-03-30T13:32:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:43 smithi187 ceph-mon[53604]: failed to encode map e94 with expected crc 2024-03-30T13:32:44.183 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:43 smithi159 podman[78245]: 2024-03-30 13:32:43.976155191 +0000 UTC m=+2.363329510 container cleanup ca0dcfe54da88164739f15aa67c1468636733bda787912d70069158f02243b27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.k8s.display-name=CentOS Stream 8, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, version=8, distribution-scope=public) 2024-03-30T13:32:44.183 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:43 smithi159 bash[78245]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2 2024-03-30T13:32:44.434 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:44 smithi159 podman[78259]: 2024-03-30 13:32:44.31540785 +0000 UTC m=+1.060215944 container remove ca0dcfe54da88164739f15aa67c1468636733bda787912d70069158f02243b27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , name=centos-stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, ceph=True, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=) 2024-03-30T13:32:44.689 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:44 smithi159 ceph-mon[67002]: osdmap e95: 8 total, 7 up, 8 in 2024-03-30T13:32:44.690 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:44 smithi159 ceph-mon[67002]: failed to encode map e95 with expected crc 2024-03-30T13:32:44.690 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:44 smithi159 ceph-mon[67002]: failed to encode map e95 with expected crc 2024-03-30T13:32:44.690 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:44 smithi159 ceph-mon[67002]: failed to encode map e95 with expected crc 2024-03-30T13:32:44.690 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:44 smithi159 ceph-mon[67002]: failed to encode map e95 with expected crc 2024-03-30T13:32:44.690 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:44 smithi159 ceph-mon[67002]: failed to encode map e95 with expected crc 2024-03-30T13:32:44.690 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:44 smithi159 ceph-mon[67002]: failed to encode map e95 with expected crc 2024-03-30T13:32:44.690 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:44 smithi159 ceph-mon[67002]: failed to encode map e95 with expected crc 2024-03-30T13:32:44.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 243723 10.66 MB/sec execute 426 sec latency 38.893 ms 2024-03-30T13:32:44.940 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:44 smithi159 podman[78345]: 2024-03-30 13:32:44.776430435 +0000 UTC m=+0.023323377 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:32:44.940 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:44 smithi159 podman[78345]: 2024-03-30 13:32:44.899938118 +0000 UTC m=+0.146831056 container create 70131bba7c6830804362e16f38bb781a91706ac5ee13aad4a8502212476383c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-30T13:32:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:44 smithi187 ceph-mon[53604]: osdmap e95: 8 total, 7 up, 8 in 2024-03-30T13:32:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:44 smithi187 ceph-mon[53604]: failed to encode map e95 with expected crc 2024-03-30T13:32:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:44 smithi187 ceph-mon[53604]: failed to encode map e95 with expected crc 2024-03-30T13:32:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:44 smithi187 ceph-mon[53604]: failed to encode map e95 with expected crc 2024-03-30T13:32:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:44 smithi187 ceph-mon[53604]: failed to encode map e95 with expected crc 2024-03-30T13:32:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:44 smithi187 ceph-mon[53604]: failed to encode map e95 with expected crc 2024-03-30T13:32:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:44 smithi187 ceph-mon[53604]: failed to encode map e95 with expected crc 2024-03-30T13:32:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:44 smithi187 ceph-mon[53604]: failed to encode map e95 with expected crc 2024-03-30T13:32:45.445 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:45 smithi159 podman[78345]: 2024-03-30 13:32:45.362222097 +0000 UTC m=+0.609115033 container init 70131bba7c6830804362e16f38bb781a91706ac5ee13aad4a8502212476383c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-30T13:32:45.445 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:45 smithi159 podman[78345]: 2024-03-30 13:32:45.368582152 +0000 UTC m=+0.615475095 container start 70131bba7c6830804362e16f38bb781a91706ac5ee13aad4a8502212476383c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:32:45.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:45 smithi159 ceph-mon[67002]: pgmap v136: 97 pgs: 18 peering, 2 stale+active+clean, 77 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.7 MiB/s wr, 324 op/s 2024-03-30T13:32:45.697 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:45 smithi159 ceph-mon[67002]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-30T13:32:45.697 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:45 smithi159 podman[78345]: 2024-03-30 13:32:45.523545853 +0000 UTC m=+0.770438813 container attach 70131bba7c6830804362e16f38bb781a91706ac5ee13aad4a8502212476383c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-30T13:32:45.919 INFO:teuthology.orchestra.run.smithi159.stdout: 5 244296 10.67 MB/sec execute 427 sec latency 34.578 ms 2024-03-30T13:32:45.949 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:45 smithi159 podman[78345]: 2024-03-30 13:32:45.775542629 +0000 UTC m=+1.022435569 container died 70131bba7c6830804362e16f38bb781a91706ac5ee13aad4a8502212476383c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-03-30T13:32:46.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:45 smithi187 ceph-mon[53604]: pgmap v136: 97 pgs: 18 peering, 2 stale+active+clean, 77 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.7 MiB/s wr, 324 op/s 2024-03-30T13:32:46.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:45 smithi187 ceph-mon[53604]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-30T13:32:46.714 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:46 smithi159 podman[78370]: 2024-03-30 13:32:46.646568999 +0000 UTC m=+0.856071522 container remove 70131bba7c6830804362e16f38bb781a91706ac5ee13aad4a8502212476383c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-03-30T13:32:46.715 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:46 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.2.service: Deactivated successfully. 2024-03-30T13:32:46.715 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:46 smithi159 systemd[1]: Stopped Ceph osd.2 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:32:46.715 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:46 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.2.service: Consumed 3min 27.907s CPU time. 2024-03-30T13:32:46.715 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:46 smithi159 ceph-mon[67002]: Health check failed: Degraded data redundancy: 32/1692 objects degraded (1.891%), 3 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:46.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 244720 10.66 MB/sec execute 428 sec latency 37.156 ms 2024-03-30T13:32:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:46 smithi187 ceph-mon[53604]: Health check failed: Degraded data redundancy: 32/1692 objects degraded (1.891%), 3 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:47.233 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:47 smithi159 systemd[1]: Starting Ceph osd.2 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:32:47.484 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:47 smithi159 podman[78467]: 2024-03-30 13:32:47.185413665 +0000 UTC m=+0.024131390 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:32:47.484 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:47 smithi159 podman[78467]: 2024-03-30 13:32:47.339251856 +0000 UTC m=+0.177969647 container create 5bd02747e25170c91ead9fb65c77ad7dfa8dd01da06c881757f95cfc200a1bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , ceph=True) 2024-03-30T13:32:47.736 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:47 smithi159 ceph-mon[67002]: pgmap v137: 97 pgs: 3 active+undersized, 3 active+undersized+degraded, 18 peering, 73 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 445 op/s; 32/1692 objects degraded (1.891%) 2024-03-30T13:32:47.737 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:47 smithi159 podman[78467]: 2024-03-30 13:32:47.676515074 +0000 UTC m=+0.515232780 container init 5bd02747e25170c91ead9fb65c77ad7dfa8dd01da06c881757f95cfc200a1bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:32:47.737 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:47 smithi159 podman[78467]: 2024-03-30 13:32:47.681691111 +0000 UTC m=+0.520408825 container start 5bd02747e25170c91ead9fb65c77ad7dfa8dd01da06c881757f95cfc200a1bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:32:47.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 245087 10.67 MB/sec execute 429 sec latency 41.611 ms 2024-03-30T13:32:48.002 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:47 smithi159 podman[78467]: 2024-03-30 13:32:47.797259242 +0000 UTC m=+0.635976956 container attach 5bd02747e25170c91ead9fb65c77ad7dfa8dd01da06c881757f95cfc200a1bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-30T13:32:48.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:47 smithi187 ceph-mon[53604]: pgmap v137: 97 pgs: 3 active+undersized, 3 active+undersized+degraded, 18 peering, 73 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 445 op/s; 32/1692 objects degraded (1.891%) 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate[78483]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 bash[78467]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 bash[78467]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 bash[78467]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 bash[78467]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 bash[78467]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 bash[78467]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 bash[78467]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate[78483]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate[78483]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate[78483]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate[78483]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-30T13:32:48.508 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate[78483]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-30T13:32:48.509 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate[78483]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-30T13:32:48.509 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:48 smithi159 podman[78467]: 2024-03-30 13:32:48.356836113 +0000 UTC m=+1.195553830 container died 5bd02747e25170c91ead9fb65c77ad7dfa8dd01da06c881757f95cfc200a1bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:32:48.923 INFO:teuthology.orchestra.run.smithi159.stdout: 5 245466 10.66 MB/sec execute 430 sec latency 48.864 ms 2024-03-30T13:32:49.539 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:49 smithi159 podman[78651]: 2024-03-30 13:32:49.307195719 +0000 UTC m=+0.943991016 container remove 5bd02747e25170c91ead9fb65c77ad7dfa8dd01da06c881757f95cfc200a1bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-activate, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-03-30T13:32:49.592 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:49 smithi159 podman[78705]: 2024-03-30 13:32:49.476299919 +0000 UTC m=+0.024983625 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:32:49.790 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:49 smithi159 ceph-mon[67002]: pgmap v138: 97 pgs: 4 active+undersized, 4 active+undersized+degraded, 18 peering, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.8 MiB/s wr, 338 op/s; 45/1683 objects degraded (2.674%) 2024-03-30T13:32:49.791 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:49 smithi159 podman[78705]: 2024-03-30 13:32:49.607192967 +0000 UTC m=+0.155876645 container create 56d8fa2c6a0f7ad9d92ea3d48a28f7550b2edc7d8074f8fa5a798c10456f4ec8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:32:49.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 245883 10.66 MB/sec execute 431 sec latency 35.134 ms 2024-03-30T13:32:50.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:49 smithi187 ceph-mon[53604]: pgmap v138: 97 pgs: 4 active+undersized, 4 active+undersized+degraded, 18 peering, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.8 MiB/s wr, 338 op/s; 45/1683 objects degraded (2.674%) 2024-03-30T13:32:50.049 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:49 smithi159 podman[78705]: 2024-03-30 13:32:49.914608789 +0000 UTC m=+0.463292477 container init 56d8fa2c6a0f7ad9d92ea3d48a28f7550b2edc7d8074f8fa5a798c10456f4ec8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-03-30T13:32:50.049 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:49 smithi159 podman[78705]: 2024-03-30 13:32:49.920754164 +0000 UTC m=+0.469437852 container start 56d8fa2c6a0f7ad9d92ea3d48a28f7550b2edc7d8074f8fa5a798c10456f4ec8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2) 2024-03-30T13:32:50.050 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:49 smithi159 ceph-osd[78722]: -- 172.21.15.159:0/1262677800 <== mon.0 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b4d0d39680 con 0x55b4d0d03000 2024-03-30T13:32:50.050 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:50 smithi159 bash[78705]: 56d8fa2c6a0f7ad9d92ea3d48a28f7550b2edc7d8074f8fa5a798c10456f4ec8 2024-03-30T13:32:50.050 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:50 smithi159 systemd[1]: Started Ceph osd.2 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:32:50.554 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:50 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[78718]: 2024-03-30T13:32:50.487+0000 7fe6a73d97c0 -1 Falling back to public interface 2024-03-30T13:32:50.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 246103 10.65 MB/sec execute 432 sec latency 43.339 ms 2024-03-30T13:32:51.313 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:51 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:51.314 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:51 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:51.314 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:51 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:32:51.314 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:51 smithi159 ceph-mon[67002]: pgmap v139: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.2 MiB/s wr, 618 op/s; 148/1647 objects degraded (8.986%) 2024-03-30T13:32:51.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:51 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:51.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:51 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:51.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:51 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:32:51.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:51 smithi187 ceph-mon[53604]: pgmap v139: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.2 MiB/s wr, 618 op/s; 148/1647 objects degraded (8.986%) 2024-03-30T13:32:51.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 246466 10.66 MB/sec execute 433 sec latency 40.785 ms 2024-03-30T13:32:52.329 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:52 smithi159 ceph-mon[67002]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-30T13:32:52.329 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:52 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 148/1647 objects degraded (8.986%), 19 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:52.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:52 smithi187 ceph-mon[53604]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-30T13:32:52.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:52 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 148/1647 objects degraded (8.986%), 19 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:52.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 246830 10.65 MB/sec execute 434 sec latency 58.731 ms 2024-03-30T13:32:53.312 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:53 smithi159 ceph-mon[67002]: pgmap v140: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 507 op/s; 153/1650 objects degraded (9.273%) 2024-03-30T13:32:53.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:53 smithi187 ceph-mon[53604]: pgmap v140: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 507 op/s; 153/1650 objects degraded (9.273%) 2024-03-30T13:32:53.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 247115 10.65 MB/sec execute 435 sec latency 42.294 ms 2024-03-30T13:32:54.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 247392 10.65 MB/sec execute 436 sec latency 52.294 ms 2024-03-30T13:32:55.375 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:55 smithi159 ceph-mon[67002]: pgmap v141: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 579 op/s; 157/1692 objects degraded (9.279%) 2024-03-30T13:32:55.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:55 smithi187 ceph-mon[53604]: pgmap v141: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 579 op/s; 157/1692 objects degraded (9.279%) 2024-03-30T13:32:55.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 247675 10.63 MB/sec execute 437 sec latency 57.299 ms 2024-03-30T13:32:56.652 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:56 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[78718]: 2024-03-30T13:32:56.555+0000 7fe6a73d97c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-03-30T13:32:56.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 248034 10.63 MB/sec execute 438 sec latency 85.989 ms 2024-03-30T13:32:57.432 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:57 smithi159 ceph-mon[67002]: pgmap v142: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 523 op/s; 156/1701 objects degraded (9.171%) 2024-03-30T13:32:57.432 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:57 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 156/1701 objects degraded (9.171%), 19 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:57.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:57 smithi187 ceph-mon[53604]: pgmap v142: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 523 op/s; 156/1701 objects degraded (9.171%) 2024-03-30T13:32:57.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:57 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 156/1701 objects degraded (9.171%), 19 pgs degraded (PG_DEGRADED) 2024-03-30T13:32:57.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 248310 10.62 MB/sec execute 439 sec latency 57.045 ms 2024-03-30T13:32:57.945 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:57 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[78718]: 2024-03-30T13:32:57.820+0000 7fe6a73d97c0 -1 osd.2 93 log_to_monitors true 2024-03-30T13:32:58.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:58.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:32:58.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:58.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:58 smithi159 ceph-mon[67002]: from='osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-30T13:32:58.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:58.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:32:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:58 smithi187 ceph-mon[53604]: from='osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-30T13:32:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:32:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 248530 10.61 MB/sec execute 440 sec latency 101.235 ms 2024-03-30T13:32:59.242 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:32:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[78718]: 2024-03-30T13:32:59.065+0000 7fe694829700 -1 osd.2 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:32:59.744 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:59 smithi159 ceph-mon[67002]: pgmap v143: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 447 op/s; 154/1689 objects degraded (9.118%) 2024-03-30T13:32:59.744 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:59 smithi159 ceph-mon[67002]: from='osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-30T13:32:59.744 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:59 smithi159 ceph-mon[67002]: failed to encode map e96 with expected crc 2024-03-30T13:32:59.745 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:59 smithi159 ceph-mon[67002]: osdmap e96: 8 total, 7 up, 8 in 2024-03-30T13:32:59.745 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:59 smithi159 ceph-mon[67002]: from='osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:32:59.745 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:32:59 smithi159 ceph-mon[67002]: from='osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935]' entity='osd.2' 2024-03-30T13:32:59.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 248839 10.60 MB/sec execute 441 sec latency 59.364 ms 2024-03-30T13:33:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:59 smithi187 ceph-mon[53604]: pgmap v143: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 447 op/s; 154/1689 objects degraded (9.118%) 2024-03-30T13:33:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:59 smithi187 ceph-mon[53604]: from='osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-30T13:33:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:59 smithi187 ceph-mon[53604]: failed to encode map e96 with expected crc 2024-03-30T13:33:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:59 smithi187 ceph-mon[53604]: osdmap e96: 8 total, 7 up, 8 in 2024-03-30T13:33:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:59 smithi187 ceph-mon[53604]: from='osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:33:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:32:59 smithi187 ceph-mon[53604]: from='osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935]' entity='osd.2' 2024-03-30T13:33:00.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 248948 10.59 MB/sec execute 442 sec latency 883.827 ms 2024-03-30T13:33:01.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: failed to encode map e97 with expected crc 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935] boot 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: osdmap e97: 8 total, 8 up, 8 in 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: failed to encode map e97 with expected crc 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: failed to encode map e97 with expected crc 2024-03-30T13:33:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:00 smithi187 ceph-mon[53604]: pgmap v146: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 582 op/s; 159/1656 objects degraded (9.601%) 2024-03-30T13:33:01.075 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:01.075 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:01.075 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:33:01.075 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: failed to encode map e97 with expected crc 2024-03-30T13:33:01.075 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: osd.2 [v2:172.21.15.159:6818/256600935,v1:172.21.15.159:6819/256600935] boot 2024-03-30T13:33:01.076 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: osdmap e97: 8 total, 8 up, 8 in 2024-03-30T13:33:01.076 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: failed to encode map e97 with expected crc 2024-03-30T13:33:01.076 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-30T13:33:01.076 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: failed to encode map e97 with expected crc 2024-03-30T13:33:01.076 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:00 smithi159 ceph-mon[67002]: pgmap v146: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 582 op/s; 159/1656 objects degraded (9.601%) 2024-03-30T13:33:01.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 248971 10.57 MB/sec execute 443 sec latency 1883.858 ms 2024-03-30T13:33:02.306 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: osdmap e98: 8 total, 8 up, 8 in 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.307 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:02 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 159/1656 objects degraded (9.601%), 19 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: osdmap e98: 8 total, 8 up, 8 in 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: failed to encode map e98 with expected crc 2024-03-30T13:33:02.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:02 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 159/1656 objects degraded (9.601%), 19 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:02.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 249291 10.56 MB/sec execute 444 sec latency 2070.476 ms 2024-03-30T13:33:03.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.086 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: osdmap e99: 8 total, 8 up, 8 in 2024-03-30T13:33:03.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.087 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:03 smithi159 ceph-mon[67002]: pgmap v149: 97 pgs: 4 peering, 5 active+undersized, 17 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.7 MiB/s wr, 533 op/s; 130/1638 objects degraded (7.937%) 2024-03-30T13:33:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: osdmap e99: 8 total, 8 up, 8 in 2024-03-30T13:33:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: failed to encode map e99 with expected crc 2024-03-30T13:33:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:03 smithi187 ceph-mon[53604]: pgmap v149: 97 pgs: 4 peering, 5 active+undersized, 17 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.7 MiB/s wr, 533 op/s; 130/1638 objects degraded (7.937%) 2024-03-30T13:33:03.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 249546 10.55 MB/sec execute 445 sec latency 650.393 ms 2024-03-30T13:33:04.119 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:04 smithi159 ceph-mon[67002]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.119 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:04 smithi159 ceph-mon[67002]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.120 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:04 smithi159 ceph-mon[67002]: osdmap e100: 8 total, 8 up, 8 in 2024-03-30T13:33:04.120 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:04 smithi159 ceph-mon[67002]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.120 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:04 smithi159 ceph-mon[67002]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.120 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:04 smithi159 ceph-mon[67002]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.120 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:04 smithi159 ceph-mon[67002]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.422 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:04 smithi187 ceph-mon[53604]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.422 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:04 smithi187 ceph-mon[53604]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.422 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:04 smithi187 ceph-mon[53604]: osdmap e100: 8 total, 8 up, 8 in 2024-03-30T13:33:04.422 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:04 smithi187 ceph-mon[53604]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.422 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:04 smithi187 ceph-mon[53604]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.422 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:04 smithi187 ceph-mon[53604]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.422 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:04 smithi187 ceph-mon[53604]: failed to encode map e100 with expected crc 2024-03-30T13:33:04.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 249740 10.54 MB/sec execute 446 sec latency 827.151 ms 2024-03-30T13:33:05.156 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: osdmap e101: 8 total, 8 up, 8 in 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.157 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:05 smithi159 ceph-mon[67002]: pgmap v152: 97 pgs: 1 active+recovery_wait+degraded, 7 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 85 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.4 MiB/s wr, 364 op/s; 52/1554 objects degraded (3.346%); 2.0 MiB/s, 20 objects/s recovering 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: osdmap e101: 8 total, 8 up, 8 in 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: failed to encode map e101 with expected crc 2024-03-30T13:33:05.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:05 smithi187 ceph-mon[53604]: pgmap v152: 97 pgs: 1 active+recovery_wait+degraded, 7 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 85 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.4 MiB/s wr, 364 op/s; 52/1554 objects degraded (3.346%); 2.0 MiB/s, 20 objects/s recovering 2024-03-30T13:33:05.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 250067 10.54 MB/sec execute 447 sec latency 667.411 ms 2024-03-30T13:33:06.206 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.206 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: osdmap e102: 8 total, 8 up, 8 in 2024-03-30T13:33:06.206 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.206 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.206 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.206 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: pgmap v154: 97 pgs: 1 active+recovery_wait+degraded, 7 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 85 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.7 MiB/s wr, 424 op/s; 52/1554 objects degraded (3.346%); 2.5 MiB/s, 24 objects/s recovering 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: pgmap v155: 97 pgs: 1 active+recovery_wait+degraded, 7 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 85 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.0 MiB/s wr, 455 op/s; 52/1554 objects degraded (3.346%); 2.7 MiB/s, 26 objects/s recovering 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:06.207 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-30T13:33:06.251 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-30T13:33:06.251 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:06 smithi159 ceph-mon[67002]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: osdmap e102: 8 total, 8 up, 8 in 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: failed to encode map e102 with expected crc 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:33:06.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: pgmap v154: 97 pgs: 1 active+recovery_wait+degraded, 7 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 85 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.7 MiB/s wr, 424 op/s; 52/1554 objects degraded (3.346%); 2.5 MiB/s, 24 objects/s recovering 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: pgmap v155: 97 pgs: 1 active+recovery_wait+degraded, 7 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 85 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.0 MiB/s wr, 455 op/s; 52/1554 objects degraded (3.346%); 2.7 MiB/s, 26 objects/s recovering 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-30T13:33:06.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:06 smithi187 ceph-mon[53604]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-30T13:33:06.750 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:33:06.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 250352 10.53 MB/sec execute 448 sec latency 48.968 ms 2024-03-30T13:33:07.491 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:07 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 52/1554 objects degraded (3.346%), 4 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:07.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:07 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 52/1554 objects degraded (3.346%), 4 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:07.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 250593 10.52 MB/sec execute 449 sec latency 56.181 ms 2024-03-30T13:33:07.935 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:33:07.935 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (6m) 9s ago 15m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (2m) 9s ago 15m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (2m) 2m ago 15m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 9s ago 15m - - 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (9m) 9s ago 9m 38.6M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (9m) 2m ago 9m 16.8M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (5m) 9s ago 17m 506M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (4m) 2m ago 15m 449M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (4m) 9s ago 17m 47.8M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (3m) 2m ago 15m 34.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (9m) 9s ago 9m 284M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (9m) 2m ago 9m 86.1M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (6m) 9s ago 15m 10.0M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (5m) 2m ago 14m 18.3M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (116s) 9s ago 14m 306M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (66s) 9s ago 13m 250M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:33:07.936 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (18s) 9s ago 13m 11.5M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:33:07.937 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (12m) 9s ago 12m 464M 1936M 17.2.0 e1d6a67b021e 4c59c75f94c0 2024-03-30T13:33:07.937 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (12m) 2m ago 12m 595M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:33:07.937 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (12m) 2m ago 12m 465M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:33:07.937 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (11m) 2m ago 11m 378M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:33:07.937 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (11m) 2m ago 11m 538M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:33:07.937 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (4m) 9s ago 14m 63.1M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:33:08.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:08 smithi159 ceph-mon[67002]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:08.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:08 smithi159 ceph-mon[67002]: pgmap v156: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 89 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.8 MiB/s wr, 317 op/s; 52/1569 objects degraded (3.314%); 2.1 MiB/s, 23 objects/s recovering 2024-03-30T13:33:08.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:08 smithi159 ceph-mon[67002]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 3 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:33:08.581 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:33:08.582 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:33:08.582 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:33:08.582 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:33:08.582 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-30T13:33:08.582 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 7 2024-03-30T13:33:08.582 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:33:08.582 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:33:08.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:08 smithi187 ceph-mon[53604]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:08.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:08 smithi187 ceph-mon[53604]: pgmap v156: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 89 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.8 MiB/s wr, 317 op/s; 52/1569 objects degraded (3.314%); 2.1 MiB/s, 23 objects/s recovering 2024-03-30T13:33:08.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:08 smithi187 ceph-mon[53604]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:08.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 250794 10.51 MB/sec execute 450 sec latency 67.530 ms 2024-03-30T13:33:09.115 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:33:09.115 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:33:09.115 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:33:09.115 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:33:09.115 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:33:09.115 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:33:09.115 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:33:09.116 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:33:09.116 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:33:09.116 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "9/23 daemons upgraded", 2024-03-30T13:33:09.116 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:33:09.116 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:33:09.116 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:33:09.539 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:09 smithi159 ceph-mon[67002]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:09.539 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:09 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3644544893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:09.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:09 smithi187 ceph-mon[53604]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:09.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:09 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3644544893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:09.825 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:33:09.825 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:33:09.825 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:33:09.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 251036 10.52 MB/sec execute 451 sec latency 66.896 ms 2024-03-30T13:33:10.570 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:10 smithi159 ceph-mon[67002]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:10.570 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:10 smithi159 ceph-mon[67002]: pgmap v157: 97 pgs: 97 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 373 op/s; 1.7 MiB/s, 19 objects/s recovering 2024-03-30T13:33:10.570 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:10 smithi159 ceph-mon[67002]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/1569 objects degraded (3.314%), 4 pgs degraded) 2024-03-30T13:33:10.571 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:10 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3698216694' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:33:10.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:10 smithi187 ceph-mon[53604]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:10.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:10 smithi187 ceph-mon[53604]: pgmap v157: 97 pgs: 97 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 373 op/s; 1.7 MiB/s, 19 objects/s recovering 2024-03-30T13:33:10.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:10 smithi187 ceph-mon[53604]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/1569 objects degraded (3.314%), 4 pgs degraded) 2024-03-30T13:33:10.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:10 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3698216694' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:33:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 251304 10.50 MB/sec execute 452 sec latency 68.304 ms 2024-03-30T13:33:11.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 251610 10.49 MB/sec execute 453 sec latency 61.584 ms 2024-03-30T13:33:12.345 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:12 smithi159 ceph-mon[67002]: pgmap v158: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.3 MiB/s wr, 289 op/s; 1.3 MiB/s, 14 objects/s recovering 2024-03-30T13:33:12.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:12 smithi187 ceph-mon[53604]: pgmap v158: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.3 MiB/s wr, 289 op/s; 1.3 MiB/s, 14 objects/s recovering 2024-03-30T13:33:12.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 251807 10.47 MB/sec execute 454 sec latency 48.839 ms 2024-03-30T13:33:13.666 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:13.667 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:33:13.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:33:13.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 252003 10.47 MB/sec execute 455 sec latency 53.071 ms 2024-03-30T13:33:14.631 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:14 smithi159 ceph-mon[67002]: pgmap v159: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 1.1 MiB/s, 12 objects/s recovering 2024-03-30T13:33:14.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:14 smithi187 ceph-mon[53604]: pgmap v159: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 1.1 MiB/s, 12 objects/s recovering 2024-03-30T13:33:14.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 252229 10.46 MB/sec execute 456 sec latency 50.213 ms 2024-03-30T13:33:15.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 252578 10.46 MB/sec execute 457 sec latency 60.433 ms 2024-03-30T13:33:16.518 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:16 smithi159 ceph-mon[67002]: pgmap v160: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 376 op/s; 900 KiB/s, 10 objects/s recovering 2024-03-30T13:33:16.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:16 smithi187 ceph-mon[53604]: pgmap v160: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 376 op/s; 900 KiB/s, 10 objects/s recovering 2024-03-30T13:33:16.924 INFO:teuthology.orchestra.run.smithi159.stdout: 5 253025 10.46 MB/sec execute 458 sec latency 50.875 ms 2024-03-30T13:33:17.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 253325 10.45 MB/sec execute 459 sec latency 43.713 ms 2024-03-30T13:33:18.578 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:18 smithi159 ceph-mon[67002]: pgmap v161: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.3 MiB/s wr, 320 op/s; 750 KiB/s, 8 objects/s recovering 2024-03-30T13:33:18.578 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:18.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 253737 10.45 MB/sec execute 460 sec latency 64.379 ms 2024-03-30T13:33:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:18 smithi187 ceph-mon[53604]: pgmap v161: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.3 MiB/s wr, 320 op/s; 750 KiB/s, 8 objects/s recovering 2024-03-30T13:33:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:19.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 254068 10.45 MB/sec execute 461 sec latency 35.024 ms 2024-03-30T13:33:20.645 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:20 smithi159 ceph-mon[67002]: pgmap v162: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 682 KiB/s, 4 objects/s recovering 2024-03-30T13:33:20.646 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-30T13:33:20.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 254344 10.45 MB/sec execute 462 sec latency 45.152 ms 2024-03-30T13:33:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:20 smithi187 ceph-mon[53604]: pgmap v162: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 682 KiB/s, 4 objects/s recovering 2024-03-30T13:33:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-30T13:33:21.655 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:21 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-30T13:33:21.655 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:21 smithi159 ceph-mon[67002]: Upgrade: osd.3 is safe to restart 2024-03-30T13:33:21.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 254632 10.45 MB/sec execute 463 sec latency 52.000 ms 2024-03-30T13:33:22.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:21 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-30T13:33:22.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:21 smithi187 ceph-mon[53604]: Upgrade: osd.3 is safe to restart 2024-03-30T13:33:22.705 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:22 smithi159 ceph-mon[67002]: pgmap v163: 97 pgs: 97 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.8 MiB/s wr, 412 op/s 2024-03-30T13:33:22.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 254973 10.44 MB/sec execute 464 sec latency 64.492 ms 2024-03-30T13:33:23.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:22 smithi187 ceph-mon[53604]: pgmap v163: 97 pgs: 97 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.8 MiB/s wr, 412 op/s 2024-03-30T13:33:23.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 255273 10.43 MB/sec execute 465 sec latency 58.303 ms 2024-03-30T13:33:24.765 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:24 smithi159 ceph-mon[67002]: pgmap v164: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.0 MiB/s wr, 420 op/s 2024-03-30T13:33:24.766 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:24 smithi159 ceph-mon[67002]: Upgrade: Updating osd.3 2024-03-30T13:33:24.766 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:24 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:24.766 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:24 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-30T13:33:24.766 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:24 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:33:24.766 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:24 smithi159 ceph-mon[67002]: Deploying daemon osd.3 on smithi159 2024-03-30T13:33:24.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 255459 10.41 MB/sec execute 466 sec latency 56.311 ms 2024-03-30T13:33:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:24 smithi187 ceph-mon[53604]: pgmap v164: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.0 MiB/s wr, 420 op/s 2024-03-30T13:33:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:24 smithi187 ceph-mon[53604]: Upgrade: Updating osd.3 2024-03-30T13:33:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:24 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:24 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-30T13:33:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:24 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:33:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:24 smithi187 ceph-mon[53604]: Deploying daemon osd.3 on smithi159 2024-03-30T13:33:25.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 255739 10.41 MB/sec execute 467 sec latency 42.696 ms 2024-03-30T13:33:26.854 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:26 smithi159 ceph-mon[67002]: pgmap v165: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.8 MiB/s wr, 497 op/s 2024-03-30T13:33:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 256069 10.41 MB/sec execute 468 sec latency 48.705 ms 2024-03-30T13:33:27.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:26 smithi187 ceph-mon[53604]: pgmap v165: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.8 MiB/s wr, 497 op/s 2024-03-30T13:33:27.618 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:27 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:33:27.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 256402 10.41 MB/sec execute 469 sec latency 55.593 ms 2024-03-30T13:33:28.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:27 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:33:28.660 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:28 smithi159 ceph-mon[67002]: pgmap v166: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-03-30T13:33:28.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 256656 10.40 MB/sec execute 470 sec latency 65.399 ms 2024-03-30T13:33:29.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:28 smithi187 ceph-mon[53604]: pgmap v166: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-03-30T13:33:29.415 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:29 smithi159 systemd[1]: Stopping Ceph osd.3 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:33:29.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 256871 10.39 MB/sec execute 471 sec latency 63.741 ms 2024-03-30T13:33:29.926 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:29 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[45730]: 2024-03-30T13:33:29.712+0000 7f0fde846700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:33:29.926 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:29 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[45730]: 2024-03-30T13:33:29.712+0000 7f0fde846700 -1 osd.3 102 *** Got signal Terminated *** 2024-03-30T13:33:29.926 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:29 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[45730]: 2024-03-30T13:33:29.712+0000 7f0fde846700 -1 osd.3 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:33:30.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:30 smithi159 ceph-mon[67002]: pgmap v167: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 560 op/s 2024-03-30T13:33:30.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:30 smithi159 ceph-mon[67002]: osd.3 marked itself down and dead 2024-03-30T13:33:30.925 INFO:teuthology.orchestra.run.smithi159.stdout: 5 257203 10.39 MB/sec execute 472 sec latency 318.119 ms 2024-03-30T13:33:31.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:30 smithi187 ceph-mon[53604]: pgmap v167: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 560 op/s 2024-03-30T13:33:31.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:30 smithi187 ceph-mon[53604]: osd.3 marked itself down and dead 2024-03-30T13:33:31.503 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:31 smithi159 podman[80822]: 2024-03-30 13:33:31.280500114 +0000 UTC m=+1.719458482 container died 4c59c75f94c04723609ae3c0bb0d16e1039edf16959680f7dc112e41455f12d4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, architecture=x86_64, vendor=Red Hat, Inc., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870) 2024-03-30T13:33:31.828 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:31 smithi159 ceph-mon[67002]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:33:31.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:31 smithi159 ceph-mon[67002]: failed to encode map e103 with expected crc 2024-03-30T13:33:31.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:31 smithi159 ceph-mon[67002]: osdmap e103: 8 total, 7 up, 8 in 2024-03-30T13:33:31.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:31 smithi159 ceph-mon[67002]: failed to encode map e104 with expected crc 2024-03-30T13:33:31.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:31 smithi159 ceph-mon[67002]: osdmap e104: 8 total, 7 up, 8 in 2024-03-30T13:33:31.833 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:31 smithi159 podman[80822]: 2024-03-30 13:33:31.792615585 +0000 UTC m=+2.231573906 container cleanup 4c59c75f94c04723609ae3c0bb0d16e1039edf16959680f7dc112e41455f12d4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, architecture=x86_64, distribution-scope=public, name=centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., release=754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD) 2024-03-30T13:33:31.833 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:31 smithi159 bash[80822]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3 2024-03-30T13:33:31.927 INFO:teuthology.orchestra.run.smithi159.stdout: 5 257330 10.37 MB/sec execute 473 sec latency 1051.689 ms 2024-03-30T13:33:32.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:31 smithi187 ceph-mon[53604]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:33:32.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:31 smithi187 ceph-mon[53604]: failed to encode map e103 with expected crc 2024-03-30T13:33:32.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:31 smithi187 ceph-mon[53604]: osdmap e103: 8 total, 7 up, 8 in 2024-03-30T13:33:32.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:31 smithi187 ceph-mon[53604]: failed to encode map e104 with expected crc 2024-03-30T13:33:32.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:31 smithi187 ceph-mon[53604]: osdmap e104: 8 total, 7 up, 8 in 2024-03-30T13:33:32.330 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:32 smithi159 podman[80838]: 2024-03-30 13:33:32.153769991 +0000 UTC m=+0.875632929 container remove 4c59c75f94c04723609ae3c0bb0d16e1039edf16959680f7dc112e41455f12d4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-type=git, release=754, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.openshift.expose-services=) 2024-03-30T13:33:32.838 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:32 smithi159 podman[80926]: 2024-03-30 13:33:32.607229576 +0000 UTC m=+0.025895975 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:33:32.838 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:32 smithi159 podman[80926]: 2024-03-30 13:33:32.788852669 +0000 UTC m=+0.207519061 container create ba525da0c3cec645ca97931276be481eac392b56ab5201482cf575b4aed28252 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, RELEASE=squid-4e50bbc, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:33:32.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:32 smithi159 ceph-mon[67002]: pgmap v169: 97 pgs: 16 stale+active+clean, 81 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.0 MiB/s wr, 376 op/s 2024-03-30T13:33:32.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:32 smithi159 ceph-mon[67002]: failed to encode map e104 with expected crc 2024-03-30T13:33:32.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:32 smithi159 ceph-mon[67002]: failed to encode map e104 with expected crc 2024-03-30T13:33:32.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:32 smithi159 ceph-mon[67002]: failed to encode map e104 with expected crc 2024-03-30T13:33:32.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:32 smithi159 ceph-mon[67002]: failed to encode map e104 with expected crc 2024-03-30T13:33:32.838 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:32 smithi159 ceph-mon[67002]: failed to encode map e104 with expected crc 2024-03-30T13:33:32.839 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:32 smithi159 ceph-mon[67002]: failed to encode map e104 with expected crc 2024-03-30T13:33:32.839 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:32 smithi159 ceph-mon[67002]: failed to encode map e104 with expected crc 2024-03-30T13:33:32.927 INFO:teuthology.orchestra.run.smithi159.stdout: 5 257748 10.37 MB/sec execute 474 sec latency 46.299 ms 2024-03-30T13:33:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:32 smithi187 ceph-mon[53604]: pgmap v169: 97 pgs: 16 stale+active+clean, 81 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.0 MiB/s wr, 376 op/s 2024-03-30T13:33:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:32 smithi187 ceph-mon[53604]: failed to encode map e104 with expected crc 2024-03-30T13:33:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:32 smithi187 ceph-mon[53604]: failed to encode map e104 with expected crc 2024-03-30T13:33:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:32 smithi187 ceph-mon[53604]: failed to encode map e104 with expected crc 2024-03-30T13:33:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:32 smithi187 ceph-mon[53604]: failed to encode map e104 with expected crc 2024-03-30T13:33:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:32 smithi187 ceph-mon[53604]: failed to encode map e104 with expected crc 2024-03-30T13:33:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:32 smithi187 ceph-mon[53604]: failed to encode map e104 with expected crc 2024-03-30T13:33:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:32 smithi187 ceph-mon[53604]: failed to encode map e104 with expected crc 2024-03-30T13:33:33.340 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:33 smithi159 podman[80926]: 2024-03-30 13:33:33.11755278 +0000 UTC m=+0.536219179 container init ba525da0c3cec645ca97931276be481eac392b56ab5201482cf575b4aed28252 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc) 2024-03-30T13:33:33.340 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:33 smithi159 podman[80926]: 2024-03-30 13:33:33.124988097 +0000 UTC m=+0.543654487 container start ba525da0c3cec645ca97931276be481eac392b56ab5201482cf575b4aed28252 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:33:33.341 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:33 smithi159 podman[80926]: 2024-03-30 13:33:33.318482437 +0000 UTC m=+0.737148826 container attach ba525da0c3cec645ca97931276be481eac392b56ab5201482cf575b4aed28252 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:33:33.591 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:33 smithi159 podman[80926]: 2024-03-30 13:33:33.563466008 +0000 UTC m=+0.982132415 container died ba525da0c3cec645ca97931276be481eac392b56ab5201482cf575b4aed28252 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS) 2024-03-30T13:33:33.842 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:33 smithi159 ceph-mon[67002]: pgmap v171: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 13 stale+active+clean, 75 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.7 MiB/s wr, 414 op/s; 63/1674 objects degraded (3.763%); 664 B/s, 0 objects/s recovering 2024-03-30T13:33:33.842 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:33 smithi159 ceph-mon[67002]: Health check failed: Degraded data redundancy: 63/1674 objects degraded (3.763%), 7 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:33.927 INFO:teuthology.orchestra.run.smithi159.stdout: 5 258157 10.39 MB/sec execute 475 sec latency 42.408 ms 2024-03-30T13:33:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:33 smithi187 ceph-mon[53604]: pgmap v171: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 13 stale+active+clean, 75 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.7 MiB/s wr, 414 op/s; 63/1674 objects degraded (3.763%); 664 B/s, 0 objects/s recovering 2024-03-30T13:33:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:33 smithi187 ceph-mon[53604]: Health check failed: Degraded data redundancy: 63/1674 objects degraded (3.763%), 7 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:34.609 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:34 smithi159 podman[80953]: 2024-03-30 13:33:34.468364057 +0000 UTC m=+0.899157278 container remove ba525da0c3cec645ca97931276be481eac392b56ab5201482cf575b4aed28252 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-03-30T13:33:34.610 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:34 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.3.service: Deactivated successfully. 2024-03-30T13:33:34.610 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:34 smithi159 systemd[1]: Stopped Ceph osd.3 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:33:34.610 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:34 smithi159 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.3.service: Consumed 5min 28.801s CPU time. 2024-03-30T13:33:34.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 258751 10.38 MB/sec execute 476 sec latency 34.515 ms 2024-03-30T13:33:35.116 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:34 smithi159 systemd[1]: Starting Ceph osd.3 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:33:35.376 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:35 smithi159 podman[81051]: 2024-03-30 13:33:35.038157219 +0000 UTC m=+0.023233471 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:33:35.376 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:35 smithi159 podman[81051]: 2024-03-30 13:33:35.172821881 +0000 UTC m=+0.157898098 container create 13ee52389603ec8f0d955b3634c6538fd4249eb7422c74336c0eb2e2eca37206 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-30T13:33:35.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 259066 10.38 MB/sec execute 477 sec latency 38.139 ms 2024-03-30T13:33:36.144 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 podman[81051]: 2024-03-30 13:33:36.112969064 +0000 UTC m=+1.098045281 container init 13ee52389603ec8f0d955b3634c6538fd4249eb7422c74336c0eb2e2eca37206 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-30T13:33:36.144 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 podman[81051]: 2024-03-30 13:33:36.118472916 +0000 UTC m=+1.103549125 container start 13ee52389603ec8f0d955b3634c6538fd4249eb7422c74336c0eb2e2eca37206 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux ) 2024-03-30T13:33:36.404 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 podman[81051]: 2024-03-30 13:33:36.273670482 +0000 UTC m=+1.258746697 container attach 13ee52389603ec8f0d955b3634c6538fd4249eb7422c74336c0eb2e2eca37206 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=) 2024-03-30T13:33:36.404 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:36 smithi159 ceph-mon[67002]: pgmap v172: 97 pgs: 10 active+undersized, 21 active+undersized+degraded, 4 stale+active+clean, 62 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 468 op/s; 179/1695 objects degraded (10.560%); 664 B/s, 0 objects/s recovering 2024-03-30T13:33:36.723 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:36 smithi187 ceph-mon[53604]: pgmap v172: 97 pgs: 10 active+undersized, 21 active+undersized+degraded, 4 stale+active+clean, 62 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 468 op/s; 179/1695 objects degraded (10.560%); 664 B/s, 0 objects/s recovering 2024-03-30T13:33:36.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 259401 10.39 MB/sec execute 478 sec latency 44.397 ms 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate[81065]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate[81065]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 bash[81051]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 bash[81051]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 bash[81051]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 bash[81051]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 bash[81051]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 bash[81051]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 bash[81051]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate[81065]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate[81065]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-30T13:33:36.942 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate[81065]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-30T13:33:36.943 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate[81065]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-30T13:33:36.943 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate[81065]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-30T13:33:36.943 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:36 smithi159 podman[81051]: 2024-03-30 13:33:36.7480452 +0000 UTC m=+1.733121414 container died 13ee52389603ec8f0d955b3634c6538fd4249eb7422c74336c0eb2e2eca37206 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD) 2024-03-30T13:33:37.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 259693 10.38 MB/sec execute 479 sec latency 57.735 ms 2024-03-30T13:33:38.485 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:38 smithi159 ceph-mon[67002]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s wr, 237 op/s; 216/1683 objects degraded (12.834%); 664 B/s, 0 objects/s recovering 2024-03-30T13:33:38.485 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:38 smithi159 podman[81200]: 2024-03-30 13:33:38.338601256 +0000 UTC m=+1.584788321 container remove 13ee52389603ec8f0d955b3634c6538fd4249eb7422c74336c0eb2e2eca37206 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-activate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD) 2024-03-30T13:33:38.741 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:38 smithi159 podman[81260]: 2024-03-30 13:33:38.489287697 +0000 UTC m=+0.022891784 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:33:38.741 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:38 smithi159 podman[81260]: 2024-03-30 13:33:38.64203089 +0000 UTC m=+0.175634958 container create addb9726df53392c51b550704d139eb981727fd696ae1029b9fc4d3b8018d64b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-30T13:33:38.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:38 smithi187 ceph-mon[53604]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s wr, 237 op/s; 216/1683 objects degraded (12.834%); 664 B/s, 0 objects/s recovering 2024-03-30T13:33:38.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 260047 10.37 MB/sec execute 480 sec latency 82.664 ms 2024-03-30T13:33:39.000 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:38 smithi159 podman[81260]: 2024-03-30 13:33:38.920319284 +0000 UTC m=+0.453923360 container init addb9726df53392c51b550704d139eb981727fd696ae1029b9fc4d3b8018d64b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc) 2024-03-30T13:33:39.000 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:38 smithi159 podman[81260]: 2024-03-30 13:33:38.926606952 +0000 UTC m=+0.460211027 container start addb9726df53392c51b550704d139eb981727fd696ae1029b9fc4d3b8018d64b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, ceph=True) 2024-03-30T13:33:39.000 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:38 smithi159 ceph-osd[81278]: -- 172.21.15.159:0/2084331358 <== mon.0 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c69decf680 con 0x55c69de99000 2024-03-30T13:33:39.253 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:39 smithi159 bash[81260]: addb9726df53392c51b550704d139eb981727fd696ae1029b9fc4d3b8018d64b 2024-03-30T13:33:39.253 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:39 smithi159 systemd[1]: Started Ceph osd.3 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:33:39.508 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:39 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[81274]: 2024-03-30T13:33:39.488+0000 7f70149847c0 -1 Falling back to public interface 2024-03-30T13:33:39.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 260267 10.36 MB/sec execute 481 sec latency 43.553 ms 2024-03-30T13:33:40.274 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:40 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:40.274 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:40 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:40.274 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:40 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:33:40.274 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:40 smithi159 ceph-mon[67002]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 570 op/s; 205/1593 objects degraded (12.869%); 610 B/s, 0 objects/s recovering 2024-03-30T13:33:40.283 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:33:40.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:40 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:40 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:40 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:33:40.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:40 smithi187 ceph-mon[53604]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 570 op/s; 205/1593 objects degraded (12.869%); 610 B/s, 0 objects/s recovering 2024-03-30T13:33:40.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 260447 10.35 MB/sec execute 482 sec latency 63.950 ms 2024-03-30T13:33:41.137 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:41 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 205/1593 objects degraded (12.869%), 27 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:41.138 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:41 smithi159 ceph-mon[67002]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:41.186 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:33:41.186 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (7m) 42s ago 16m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:33:41.186 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (3m) 42s ago 16m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:33:41.186 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (3m) 2m ago 15m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 42s ago 16m - - 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (10m) 42s ago 10m 38.6M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (10m) 2m ago 10m 16.8M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (5m) 42s ago 17m 506M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (5m) 2m ago 15m 449M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (4m) 42s ago 17m 47.8M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (4m) 2m ago 15m 34.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (10m) 42s ago 10m 284M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (10m) 2m ago 10m 86.1M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (6m) 42s ago 16m 10.0M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (6m) 2m ago 15m 18.3M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (2m) 42s ago 14m 306M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (99s) 42s ago 14m 250M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:33:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (51s) 42s ago 13m 11.5M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:33:41.188 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 starting - - - 1936M 2024-03-30T13:33:41.188 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (13m) 2m ago 13m 595M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:33:41.188 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (12m) 2m ago 12m 465M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:33:41.188 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (12m) 2m ago 12m 378M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:33:41.188 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (11m) 2m ago 11m 538M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:33:41.188 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (5m) 42s ago 15m 63.1M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:33:41.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:41 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 205/1593 objects degraded (12.869%), 27 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:41.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:41 smithi187 ceph-mon[53604]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 3 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:33:41.727 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:33:41.728 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:33:41.728 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:33:41.728 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:33:41.728 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-30T13:33:41.728 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 7 2024-03-30T13:33:41.728 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:33:41.728 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:33:41.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 260749 10.35 MB/sec execute 483 sec latency 64.353 ms 2024-03-30T13:33:42.188 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:33:42.188 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:33:42.188 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:33:42.188 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:33:42.188 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:33:42.188 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:33:42.188 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:33:42.188 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:33:42.189 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:33:42.189 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "9/23 daemons upgraded", 2024-03-30T13:33:42.189 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:33:42.189 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:33:42.189 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:33:42.329 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:42 smithi159 ceph-mon[67002]: from='client.34330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:42.329 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:42 smithi159 ceph-mon[67002]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:42.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:42 smithi159 ceph-mon[67002]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 496 op/s; 201/1590 objects degraded (12.642%); 531 B/s, 0 objects/s recovering 2024-03-30T13:33:42.330 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:42 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/71728514' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:42.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:42 smithi187 ceph-mon[53604]: from='client.34330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:42.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:42 smithi187 ceph-mon[53604]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:42.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:42 smithi187 ceph-mon[53604]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 496 op/s; 201/1590 objects degraded (12.642%); 531 B/s, 0 objects/s recovering 2024-03-30T13:33:42.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:42 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/71728514' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:42.884 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 201/1590 objects degraded (12.642%), 27 pgs degraded 2024-03-30T13:33:42.884 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:33:42.884 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:33:42.884 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-30T13:33:42.884 INFO:teuthology.orchestra.run.smithi159.stdout: osd.3 (root=default,host=smithi159) is down 2024-03-30T13:33:42.884 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 201/1590 objects degraded (12.642%), 27 pgs degraded 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-03-30T13:33:42.885 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-03-30T13:33:42.886 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-03-30T13:33:42.928 INFO:teuthology.orchestra.run.smithi159.stdout: 5 261021 10.34 MB/sec execute 484 sec latency 69.708 ms 2024-03-30T13:33:43.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:43 smithi159 ceph-mon[67002]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:43.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:43.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:33:43.600 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:43 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3613129513' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:33:43.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:43 smithi187 ceph-mon[53604]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:33:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:33:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:43 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3613129513' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:33:43.929 INFO:teuthology.orchestra.run.smithi159.stdout: 5 261239 10.33 MB/sec execute 485 sec latency 64.318 ms 2024-03-30T13:33:44.660 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:44 smithi159 ceph-mon[67002]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 438 op/s; 207/1590 objects degraded (13.019%); 454 B/s, 0 objects/s recovering 2024-03-30T13:33:44.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:44 smithi187 ceph-mon[53604]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 438 op/s; 207/1590 objects degraded (13.019%); 454 B/s, 0 objects/s recovering 2024-03-30T13:33:44.929 INFO:teuthology.orchestra.run.smithi159.stdout: 5 261457 10.33 MB/sec execute 486 sec latency 89.312 ms 2024-03-30T13:33:45.685 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:45 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 220/1647 objects degraded (13.358%), 27 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:45.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 261790 10.33 MB/sec execute 487 sec latency 52.571 ms 2024-03-30T13:33:46.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:45 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 220/1647 objects degraded (13.358%), 27 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:46.703 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:46 smithi159 ceph-mon[67002]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 497 op/s; 220/1647 objects degraded (13.358%) 2024-03-30T13:33:46.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 262164 10.32 MB/sec execute 488 sec latency 57.405 ms 2024-03-30T13:33:47.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:46 smithi187 ceph-mon[53604]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 497 op/s; 220/1647 objects degraded (13.358%) 2024-03-30T13:33:47.468 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:47 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[81274]: 2024-03-30T13:33:47.237+0000 7f70149847c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-03-30T13:33:47.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 262421 10.31 MB/sec execute 489 sec latency 43.020 ms 2024-03-30T13:33:47.991 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:47.991 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:47.992 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:47 smithi159 ceph-mon[67002]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 428 op/s; 223/1644 objects degraded (13.564%) 2024-03-30T13:33:48.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:48.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:48.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:47 smithi187 ceph-mon[53604]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 428 op/s; 223/1644 objects degraded (13.564%) 2024-03-30T13:33:48.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 262557 10.30 MB/sec execute 490 sec latency 99.518 ms 2024-03-30T13:33:49.018 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[81274]: 2024-03-30T13:33:48.846+0000 7f70149847c0 -1 osd.3 102 log_to_monitors true 2024-03-30T13:33:49.795 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:49.795 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:49.795 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:49 smithi159 ceph-mon[67002]: from='osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-30T13:33:49.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 262725 10.29 MB/sec execute 491 sec latency 116.244 ms 2024-03-30T13:33:50.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:50.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:50.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:49 smithi187 ceph-mon[53604]: from='osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-30T13:33:50.305 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:33:50 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[81274]: 2024-03-30T13:33:50.122+0000 7f7001dd4700 -1 osd.3 102 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:33:50.831 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:50 smithi159 ceph-mon[67002]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.6 MiB/s wr, 587 op/s; 225/1695 objects degraded (13.274%) 2024-03-30T13:33:50.831 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:50 smithi159 ceph-mon[67002]: from='osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-30T13:33:50.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:50 smithi159 ceph-mon[67002]: osdmap e105: 8 total, 7 up, 8 in 2024-03-30T13:33:50.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:50 smithi159 ceph-mon[67002]: failed to encode map e105 with expected crc 2024-03-30T13:33:50.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:50 smithi159 ceph-mon[67002]: from='osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:33:50.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:50 smithi159 ceph-mon[67002]: failed to encode map e105 with expected crc 2024-03-30T13:33:50.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:50 smithi159 ceph-mon[67002]: from='osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274]' entity='osd.3' 2024-03-30T13:33:50.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 263044 10.29 MB/sec execute 492 sec latency 55.641 ms 2024-03-30T13:33:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:50 smithi187 ceph-mon[53604]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.6 MiB/s wr, 587 op/s; 225/1695 objects degraded (13.274%) 2024-03-30T13:33:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:50 smithi187 ceph-mon[53604]: from='osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-30T13:33:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:50 smithi187 ceph-mon[53604]: osdmap e105: 8 total, 7 up, 8 in 2024-03-30T13:33:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:50 smithi187 ceph-mon[53604]: failed to encode map e105 with expected crc 2024-03-30T13:33:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:50 smithi187 ceph-mon[53604]: from='osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-03-30T13:33:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:50 smithi187 ceph-mon[53604]: failed to encode map e105 with expected crc 2024-03-30T13:33:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:50 smithi187 ceph-mon[53604]: from='osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274]' entity='osd.3' 2024-03-30T13:33:51.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:51 smithi159 ceph-mon[67002]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:33:51.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:51 smithi159 ceph-mon[67002]: failed to encode map e106 with expected crc 2024-03-30T13:33:51.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:51 smithi159 ceph-mon[67002]: osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274] boot 2024-03-30T13:33:51.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:51 smithi159 ceph-mon[67002]: osdmap e106: 8 total, 8 up, 8 in 2024-03-30T13:33:51.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:51 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:33:51.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 263170 10.27 MB/sec execute 493 sec latency 793.802 ms 2024-03-30T13:33:52.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:51 smithi187 ceph-mon[53604]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:33:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:51 smithi187 ceph-mon[53604]: failed to encode map e106 with expected crc 2024-03-30T13:33:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:51 smithi187 ceph-mon[53604]: osd.3 [v2:172.21.15.159:6826/120227274,v1:172.21.15.159:6827/120227274] boot 2024-03-30T13:33:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:51 smithi187 ceph-mon[53604]: osdmap e106: 8 total, 8 up, 8 in 2024-03-30T13:33:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:51 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-30T13:33:52.874 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:52 smithi159 ceph-mon[67002]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 475 op/s; 218/1677 objects degraded (12.999%) 2024-03-30T13:33:52.875 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:52 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 225/1695 objects degraded (13.274%), 27 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:52.875 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:52 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:52.875 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:52 smithi159 ceph-mon[67002]: osdmap e107: 8 total, 8 up, 8 in 2024-03-30T13:33:52.875 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:52 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:52.875 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:52 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:52.875 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:52 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:52.875 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:52 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:52.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 263170 10.25 MB/sec execute 494 sec latency 1793.827 ms 2024-03-30T13:33:53.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:52 smithi187 ceph-mon[53604]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 475 op/s; 218/1677 objects degraded (12.999%) 2024-03-30T13:33:53.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:52 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 225/1695 objects degraded (13.274%), 27 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:52 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:52 smithi187 ceph-mon[53604]: osdmap e107: 8 total, 8 up, 8 in 2024-03-30T13:33:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:52 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:52 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:52 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:52 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.683 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: failed to encode map e107 with expected crc 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: osdmap e108: 8 total, 8 up, 8 in 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:53.684 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:53.685 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:53 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-30T13:33:53.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 263434 10.25 MB/sec execute 495 sec latency 1908.560 ms 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: failed to encode map e107 with expected crc 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: osdmap e108: 8 total, 8 up, 8 in 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:33:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:33:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:33:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:33:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:53 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-30T13:33:54.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: failed to encode map e108 with expected crc 2024-03-30T13:33:54.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: failed to encode map e108 with expected crc 2024-03-30T13:33:54.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: failed to encode map e108 with expected crc 2024-03-30T13:33:54.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: pgmap v185: 97 pgs: 4 remapped+peering, 12 active+undersized, 22 active+undersized+degraded, 59 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 872 KiB/s wr, 43 op/s; 160/1644 objects degraded (9.732%); 167 KiB/s, 4 objects/s recovering 2024-03-30T13:33:54.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: pgmap v186: 97 pgs: 4 remapped+peering, 12 active+undersized, 22 active+undersized+degraded, 59 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 930 KiB/s wr, 45 op/s; 160/1644 objects degraded (9.732%); 178 KiB/s, 4 objects/s recovering 2024-03-30T13:33:54.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-30T13:33:54.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-30T13:33:54.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: osdmap e109: 8 total, 8 up, 8 in 2024-03-30T13:33:54.714 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:54 smithi159 ceph-mon[67002]: failed to encode map e109 with expected crc 2024-03-30T13:33:54.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 263730 10.24 MB/sec execute 496 sec latency 60.012 ms 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: failed to encode map e108 with expected crc 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: failed to encode map e108 with expected crc 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: failed to encode map e108 with expected crc 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: pgmap v185: 97 pgs: 4 remapped+peering, 12 active+undersized, 22 active+undersized+degraded, 59 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 872 KiB/s wr, 43 op/s; 160/1644 objects degraded (9.732%); 167 KiB/s, 4 objects/s recovering 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: pgmap v186: 97 pgs: 4 remapped+peering, 12 active+undersized, 22 active+undersized+degraded, 59 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 930 KiB/s wr, 45 op/s; 160/1644 objects degraded (9.732%); 178 KiB/s, 4 objects/s recovering 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: osdmap e109: 8 total, 8 up, 8 in 2024-03-30T13:33:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:54 smithi187 ceph-mon[53604]: failed to encode map e109 with expected crc 2024-03-30T13:33:55.720 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:55 smithi159 ceph-mon[67002]: failed to encode map e109 with expected crc 2024-03-30T13:33:55.720 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:55 smithi159 ceph-mon[67002]: failed to encode map e109 with expected crc 2024-03-30T13:33:55.720 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:55 smithi159 ceph-mon[67002]: failed to encode map e110 with expected crc 2024-03-30T13:33:55.720 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:55 smithi159 ceph-mon[67002]: osdmap e110: 8 total, 8 up, 8 in 2024-03-30T13:33:55.720 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:55 smithi159 ceph-mon[67002]: failed to encode map e110 with expected crc 2024-03-30T13:33:55.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 264007 10.23 MB/sec execute 497 sec latency 762.491 ms 2024-03-30T13:33:56.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:55 smithi187 ceph-mon[53604]: failed to encode map e109 with expected crc 2024-03-30T13:33:56.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:55 smithi187 ceph-mon[53604]: failed to encode map e109 with expected crc 2024-03-30T13:33:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:55 smithi187 ceph-mon[53604]: failed to encode map e110 with expected crc 2024-03-30T13:33:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:55 smithi187 ceph-mon[53604]: osdmap e110: 8 total, 8 up, 8 in 2024-03-30T13:33:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:55 smithi187 ceph-mon[53604]: failed to encode map e110 with expected crc 2024-03-30T13:33:56.743 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:56 smithi159 ceph-mon[67002]: failed to encode map e110 with expected crc 2024-03-30T13:33:56.743 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:56 smithi159 ceph-mon[67002]: failed to encode map e110 with expected crc 2024-03-30T13:33:56.743 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:56 smithi159 ceph-mon[67002]: pgmap v189: 97 pgs: 4 remapped+peering, 2 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 84 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 MiB/s wr, 353 op/s; 76/1539 objects degraded (4.938%); 293 KiB/s, 38 objects/s recovering 2024-03-30T13:33:56.744 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:56 smithi159 ceph-mon[67002]: osdmap e111: 8 total, 8 up, 8 in 2024-03-30T13:33:56.744 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:56 smithi159 ceph-mon[67002]: failed to encode map e111 with expected crc 2024-03-30T13:33:56.744 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:56 smithi159 ceph-mon[67002]: failed to encode map e111 with expected crc 2024-03-30T13:33:56.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 264249 10.22 MB/sec execute 498 sec latency 830.504 ms 2024-03-30T13:33:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:56 smithi187 ceph-mon[53604]: failed to encode map e110 with expected crc 2024-03-30T13:33:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:56 smithi187 ceph-mon[53604]: failed to encode map e110 with expected crc 2024-03-30T13:33:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:56 smithi187 ceph-mon[53604]: pgmap v189: 97 pgs: 4 remapped+peering, 2 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 84 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 MiB/s wr, 353 op/s; 76/1539 objects degraded (4.938%); 293 KiB/s, 38 objects/s recovering 2024-03-30T13:33:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:56 smithi187 ceph-mon[53604]: osdmap e111: 8 total, 8 up, 8 in 2024-03-30T13:33:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:56 smithi187 ceph-mon[53604]: failed to encode map e111 with expected crc 2024-03-30T13:33:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:56 smithi187 ceph-mon[53604]: failed to encode map e111 with expected crc 2024-03-30T13:33:57.776 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: failed to encode map e111 with expected crc 2024-03-30T13:33:57.776 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: failed to encode map e111 with expected crc 2024-03-30T13:33:57.776 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: failed to encode map e111 with expected crc 2024-03-30T13:33:57.776 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 76/1539 objects degraded (4.938%), 7 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:57.777 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: failed to encode map e112 with expected crc 2024-03-30T13:33:57.777 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: osdmap e112: 8 total, 8 up, 8 in 2024-03-30T13:33:57.777 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: failed to encode map e112 with expected crc 2024-03-30T13:33:57.777 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:57.777 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:57 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:33:57.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 264483 10.22 MB/sec execute 499 sec latency 811.896 ms 2024-03-30T13:33:58.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: failed to encode map e111 with expected crc 2024-03-30T13:33:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: failed to encode map e111 with expected crc 2024-03-30T13:33:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: failed to encode map e111 with expected crc 2024-03-30T13:33:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 76/1539 objects degraded (4.938%), 7 pgs degraded (PG_DEGRADED) 2024-03-30T13:33:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: failed to encode map e112 with expected crc 2024-03-30T13:33:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: osdmap e112: 8 total, 8 up, 8 in 2024-03-30T13:33:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: failed to encode map e112 with expected crc 2024-03-30T13:33:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:33:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:57 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:33:58.801 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:58 smithi159 ceph-mon[67002]: failed to encode map e112 with expected crc 2024-03-30T13:33:58.801 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:58 smithi159 ceph-mon[67002]: failed to encode map e112 with expected crc 2024-03-30T13:33:58.802 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:58 smithi159 ceph-mon[67002]: failed to encode map e112 with expected crc 2024-03-30T13:33:58.802 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:58 smithi159 ceph-mon[67002]: pgmap v192: 97 pgs: 4 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.7 MiB/s wr, 283 op/s; 55/1542 objects degraded (3.567%); 80 KiB/s, 32 objects/s recovering 2024-03-30T13:33:58.802 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:58 smithi159 ceph-mon[67002]: osdmap e113: 8 total, 8 up, 8 in 2024-03-30T13:33:58.802 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:58 smithi159 ceph-mon[67002]: failed to encode map e113 with expected crc 2024-03-30T13:33:58.802 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:58 smithi159 ceph-mon[67002]: failed to encode map e113 with expected crc 2024-03-30T13:33:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 264831 10.22 MB/sec execute 500 sec latency 904.974 ms 2024-03-30T13:33:59.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:58 smithi187 ceph-mon[53604]: failed to encode map e112 with expected crc 2024-03-30T13:33:59.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:58 smithi187 ceph-mon[53604]: failed to encode map e112 with expected crc 2024-03-30T13:33:59.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:58 smithi187 ceph-mon[53604]: failed to encode map e112 with expected crc 2024-03-30T13:33:59.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:58 smithi187 ceph-mon[53604]: pgmap v192: 97 pgs: 4 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.7 MiB/s wr, 283 op/s; 55/1542 objects degraded (3.567%); 80 KiB/s, 32 objects/s recovering 2024-03-30T13:33:59.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:58 smithi187 ceph-mon[53604]: osdmap e113: 8 total, 8 up, 8 in 2024-03-30T13:33:59.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:58 smithi187 ceph-mon[53604]: failed to encode map e113 with expected crc 2024-03-30T13:33:59.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:58 smithi187 ceph-mon[53604]: failed to encode map e113 with expected crc 2024-03-30T13:33:59.820 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:59 smithi159 ceph-mon[67002]: failed to encode map e113 with expected crc 2024-03-30T13:33:59.820 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:33:59 smithi159 ceph-mon[67002]: failed to encode map e113 with expected crc 2024-03-30T13:33:59.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 265167 10.22 MB/sec execute 501 sec latency 50.298 ms 2024-03-30T13:34:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:59 smithi187 ceph-mon[53604]: failed to encode map e113 with expected crc 2024-03-30T13:34:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:33:59 smithi187 ceph-mon[53604]: failed to encode map e113 with expected crc 2024-03-30T13:34:00.848 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:00 smithi159 ceph-mon[67002]: pgmap v194: 97 pgs: 2 peering, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.1 MiB/s wr, 200 op/s; 25/1599 objects degraded (1.563%); 1008 KiB/s, 25 objects/s recovering 2024-03-30T13:34:00.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 265519 10.21 MB/sec execute 502 sec latency 44.460 ms 2024-03-30T13:34:01.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:00 smithi187 ceph-mon[53604]: pgmap v194: 97 pgs: 2 peering, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.1 MiB/s wr, 200 op/s; 25/1599 objects degraded (1.563%); 1008 KiB/s, 25 objects/s recovering 2024-03-30T13:34:01.853 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:01 smithi159 ceph-mon[67002]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1599 objects degraded (1.563%), 1 pg degraded) 2024-03-30T13:34:01.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 265783 10.20 MB/sec execute 503 sec latency 54.494 ms 2024-03-30T13:34:02.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:01 smithi187 ceph-mon[53604]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1599 objects degraded (1.563%), 1 pg degraded) 2024-03-30T13:34:02.871 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:02 smithi159 ceph-mon[67002]: pgmap v195: 97 pgs: 2 peering, 95 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.2 MiB/s wr, 474 op/s; 720 KiB/s, 26 objects/s recovering 2024-03-30T13:34:02.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 266145 10.21 MB/sec execute 504 sec latency 44.885 ms 2024-03-30T13:34:03.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:02 smithi187 ceph-mon[53604]: pgmap v195: 97 pgs: 2 peering, 95 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.2 MiB/s wr, 474 op/s; 720 KiB/s, 26 objects/s recovering 2024-03-30T13:34:03.893 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:03 smithi159 ceph-mon[67002]: pgmap v196: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.7 MiB/s wr, 402 op/s; 592 KiB/s, 18 objects/s recovering 2024-03-30T13:34:03.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 266495 10.21 MB/sec execute 505 sec latency 63.294 ms 2024-03-30T13:34:04.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:03 smithi187 ceph-mon[53604]: pgmap v196: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.7 MiB/s wr, 402 op/s; 592 KiB/s, 18 objects/s recovering 2024-03-30T13:34:04.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 266819 10.21 MB/sec execute 506 sec latency 50.569 ms 2024-03-30T13:34:05.931 INFO:teuthology.orchestra.run.smithi159.stdout: 5 267127 10.20 MB/sec execute 507 sec latency 69.079 ms 2024-03-30T13:34:06.445 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:06 smithi159 ceph-mon[67002]: pgmap v197: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 483 op/s; 520 KiB/s, 16 objects/s recovering 2024-03-30T13:34:06.723 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:06 smithi187 ceph-mon[53604]: pgmap v197: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 483 op/s; 520 KiB/s, 16 objects/s recovering 2024-03-30T13:34:06.931 INFO:teuthology.orchestra.run.smithi159.stdout: 5 267404 10.19 MB/sec execute 508 sec latency 34.695 ms 2024-03-30T13:34:07.930 INFO:teuthology.orchestra.run.smithi159.stdout: 5 267663 10.18 MB/sec execute 509 sec latency 61.987 ms 2024-03-30T13:34:08.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:08 smithi159 ceph-mon[67002]: pgmap v198: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.6 MiB/s wr, 453 op/s; 431 KiB/s, 13 objects/s recovering 2024-03-30T13:34:08.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:08 smithi187 ceph-mon[53604]: pgmap v198: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.6 MiB/s wr, 453 op/s; 431 KiB/s, 13 objects/s recovering 2024-03-30T13:34:08.931 INFO:teuthology.orchestra.run.smithi159.stdout: 5 267995 10.18 MB/sec execute 510 sec latency 58.306 ms 2024-03-30T13:34:09.463 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-30T13:34:09.463 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:09 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-30T13:34:09.463 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:09 smithi187 ceph-mon[53604]: Upgrade: osd.4 is safe to restart 2024-03-30T13:34:09.541 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-30T13:34:09.541 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:09 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-30T13:34:09.541 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:09 smithi159 ceph-mon[67002]: Upgrade: osd.4 is safe to restart 2024-03-30T13:34:09.931 INFO:teuthology.orchestra.run.smithi159.stdout: 5 268293 10.18 MB/sec execute 511 sec latency 59.018 ms 2024-03-30T13:34:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:10 smithi187 ceph-mon[53604]: pgmap v199: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 500 op/s; 726 B/s, 7 objects/s recovering 2024-03-30T13:34:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-30T13:34:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:34:10.561 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:10 smithi159 ceph-mon[67002]: pgmap v199: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 500 op/s; 726 B/s, 7 objects/s recovering 2024-03-30T13:34:10.561 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:10.561 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-30T13:34:10.561 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:34:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 268628 10.18 MB/sec execute 512 sec latency 105.887 ms 2024-03-30T13:34:11.372 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:11 smithi159 ceph-mon[67002]: Upgrade: Updating osd.4 2024-03-30T13:34:11.372 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:11 smithi159 ceph-mon[67002]: Deploying daemon osd.4 on smithi187 2024-03-30T13:34:11.648 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:11 smithi187 ceph-mon[53604]: Upgrade: Updating osd.4 2024-03-30T13:34:11.648 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:11 smithi187 ceph-mon[53604]: Deploying daemon osd.4 on smithi187 2024-03-30T13:34:11.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 268918 10.17 MB/sec execute 513 sec latency 56.421 ms 2024-03-30T13:34:12.431 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:12 smithi159 ceph-mon[67002]: pgmap v200: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.7 MiB/s wr, 514 op/s; 682 B/s, 4 objects/s recovering 2024-03-30T13:34:12.508 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:12 smithi187 ceph-mon[53604]: pgmap v200: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.7 MiB/s wr, 514 op/s; 682 B/s, 4 objects/s recovering 2024-03-30T13:34:12.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 269241 10.16 MB/sec execute 514 sec latency 47.140 ms 2024-03-30T13:34:13.392 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:34:13.629 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:13.629 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:34:13.629 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:34:13.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:13.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 269465 10.15 MB/sec execute 515 sec latency 101.012 ms 2024-03-30T13:34:14.393 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:34:14.393 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (7m) 27s ago 16m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:34:14.393 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (3m) 27s ago 16m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (3m) 3m ago 16m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 27s ago 16m - - 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (11m) 27s ago 11m 38.5M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (11m) 3m ago 11m 16.8M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (6m) 27s ago 18m 507M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (5m) 3m ago 16m 449M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (5m) 27s ago 18m 49.7M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (4m) 3m ago 16m 34.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (10m) 27s ago 10m 283M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (10m) 3m ago 10m 86.1M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (7m) 27s ago 16m 13.0M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (6m) 3m ago 16m 18.3M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (3m) 27s ago 15m 356M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (2m) 27s ago 14m 277M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:34:14.394 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (84s) 27s ago 14m 241M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:34:14.395 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (35s) 27s ago 14m 12.0M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:34:14.395 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (13m) 3m ago 13m 595M 2704M 17.2.0 e1d6a67b021e a06aa3bcca6f 2024-03-30T13:34:14.395 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (13m) 3m ago 13m 465M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:34:14.395 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (12m) 3m ago 12m 378M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:34:14.395 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (12m) 3m ago 12m 538M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:34:14.395 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (5m) 27s ago 15m 63.2M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:34:14.633 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:14 smithi159 ceph-mon[67002]: pgmap v201: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 361 op/s 2024-03-30T13:34:14.633 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:14 smithi159 ceph-mon[67002]: from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:14.678 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:14 smithi187 systemd[1]: Stopping Ceph osd.4 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:34:14.678 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:14 smithi187 ceph-mon[53604]: pgmap v201: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 361 op/s 2024-03-30T13:34:14.678 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:14 smithi187 ceph-mon[53604]: from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:14.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 269768 10.16 MB/sec execute 516 sec latency 66.994 ms 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 4 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:34:15.011 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:34:15.012 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:34:15.012 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:34:15.012 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-30T13:34:15.012 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 8 2024-03-30T13:34:15.012 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:34:15.012 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:34:15.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:14 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[31753]: 2024-03-30T13:34:14.675+0000 7f84f55df700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:34:15.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:14 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[31753]: 2024-03-30T13:34:14.675+0000 7f84f55df700 -1 osd.4 113 *** Got signal Terminated *** 2024-03-30T13:34:15.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:14 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[31753]: 2024-03-30T13:34:14.675+0000 7f84f55df700 -1 osd.4 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "10/23 daemons upgraded", 2024-03-30T13:34:15.547 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:34:15.548 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:34:15.548 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:34:15.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:15 smithi159 ceph-mon[67002]: from='client.44269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:15.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:15 smithi159 ceph-mon[67002]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:15.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:15 smithi159 ceph-mon[67002]: osd.4 marked itself down and dead 2024-03-30T13:34:15.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:15 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/2247076498' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:15.934 INFO:teuthology.orchestra.run.smithi159.stdout: 5 269999 10.16 MB/sec execute 517 sec latency 345.023 ms 2024-03-30T13:34:16.003 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down 2024-03-30T13:34:16.003 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:34:16.003 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:34:16.003 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-30T13:34:16.003 INFO:teuthology.orchestra.run.smithi159.stdout: osd.4 (root=default,host=smithi187) is down 2024-03-30T13:34:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:15 smithi187 ceph-mon[53604]: from='client.44269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:15 smithi187 ceph-mon[53604]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:15 smithi187 ceph-mon[53604]: osd.4 marked itself down and dead 2024-03-30T13:34:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:15 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/2247076498' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:16.336 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:16 smithi187 podman[58079]: 2024-03-30 13:34:16.034030351 +0000 UTC m=+1.491864118 container died a06aa3bcca6f3647dd6c52d8913bad59f77c6b617dfd63ef4fe0b8e991efc291 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, release=754, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_BRANCH=HEAD, distribution-scope=public, io.buildah.version=1.19.8, RELEASE=HEAD, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, version=8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870) 2024-03-30T13:34:16.602 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:16 smithi187 podman[58079]: 2024-03-30 13:34:16.436646412 +0000 UTC m=+1.894480208 container cleanup a06aa3bcca6f3647dd6c52d8913bad59f77c6b617dfd63ef4fe0b8e991efc291 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_CLEAN=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, architecture=x86_64, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vendor=Red Hat, Inc.) 2024-03-30T13:34:16.602 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:16 smithi187 bash[58079]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4 2024-03-30T13:34:16.851 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: pgmap v202: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: osdmap e114: 8 total, 7 up, 8 in 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.852 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:16 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/660024452' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:34:16.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 270076 10.14 MB/sec execute 518 sec latency 1061.556 ms 2024-03-30T13:34:16.994 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:16 smithi187 podman[58097]: 2024-03-30 13:34:16.638103003 +0000 UTC m=+0.604790767 container remove a06aa3bcca6f3647dd6c52d8913bad59f77c6b617dfd63ef4fe0b8e991efc291 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, GIT_CLEAN=True, vcs-type=git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, release=754, ceph=True, io.openshift.tags=base centos centos-stream, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8) 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: pgmap v202: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: osdmap e114: 8 total, 7 up, 8 in 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: failed to encode map e114 with expected crc 2024-03-30T13:34:16.995 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:16 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/660024452' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:34:17.246 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:16 smithi187 podman[58190]: 2024-03-30 13:34:16.896941898 +0000 UTC m=+0.019296930 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:34:17.246 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:17 smithi187 podman[58190]: 2024-03-30 13:34:17.014074845 +0000 UTC m=+0.136429867 container create 68789a0a0670113ee835a5861aeb3b3a6666ecb61985e792845de77edd287941 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326) 2024-03-30T13:34:17.246 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:17 smithi187 podman[58190]: 2024-03-30 13:34:17.242032736 +0000 UTC m=+0.364387758 container init 68789a0a0670113ee835a5861aeb3b3a6666ecb61985e792845de77edd287941 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc) 2024-03-30T13:34:17.246 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:17 smithi187 podman[58190]: 2024-03-30 13:34:17.245503394 +0000 UTC m=+0.367858417 container start 68789a0a0670113ee835a5861aeb3b3a6666ecb61985e792845de77edd287941 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-30T13:34:17.522 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:17 smithi187 podman[58190]: 2024-03-30 13:34:17.334044425 +0000 UTC m=+0.456399461 container attach 68789a0a0670113ee835a5861aeb3b3a6666ecb61985e792845de77edd287941 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326) 2024-03-30T13:34:17.873 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:17 smithi187 ceph-mon[53604]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.873 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:17 smithi187 ceph-mon[53604]: osdmap e115: 8 total, 7 up, 8 in 2024-03-30T13:34:17.873 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:17 smithi187 ceph-mon[53604]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.873 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:17 smithi187 ceph-mon[53604]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.873 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:17 smithi187 ceph-mon[53604]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.873 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:17 smithi187 ceph-mon[53604]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.873 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:17 smithi187 ceph-mon[53604]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.873 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:17 smithi187 ceph-mon[53604]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.873 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:17 smithi187 podman[58190]: 2024-03-30 13:34:17.565369085 +0000 UTC m=+0.687724101 container died 68789a0a0670113ee835a5861aeb3b3a6666ecb61985e792845de77edd287941 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, org.label-schema.build-date=20240326, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-30T13:34:17.879 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:17 smithi159 ceph-mon[67002]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.879 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:17 smithi159 ceph-mon[67002]: osdmap e115: 8 total, 7 up, 8 in 2024-03-30T13:34:17.879 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:17 smithi159 ceph-mon[67002]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:17 smithi159 ceph-mon[67002]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:17 smithi159 ceph-mon[67002]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:17 smithi159 ceph-mon[67002]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:17 smithi159 ceph-mon[67002]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:17 smithi159 ceph-mon[67002]: failed to encode map e115 with expected crc 2024-03-30T13:34:17.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 270425 10.14 MB/sec execute 519 sec latency 43.195 ms 2024-03-30T13:34:18.432 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 podman[58215]: 2024-03-30 13:34:18.180871852 +0000 UTC m=+0.609856358 container remove 68789a0a0670113ee835a5861aeb3b3a6666ecb61985e792845de77edd287941 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-03-30T13:34:18.432 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.4.service: Deactivated successfully. 2024-03-30T13:34:18.432 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 systemd[1]: Stopped Ceph osd.4 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:34:18.432 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.4.service: Consumed 4min 59.241s CPU time. 2024-03-30T13:34:18.432 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 systemd[1]: Starting Ceph osd.4 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:34:18.750 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:18 smithi159 ceph-mon[67002]: pgmap v205: 97 pgs: 7 peering, 12 stale+active+clean, 78 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.9 MiB/s wr, 416 op/s 2024-03-30T13:34:18.934 INFO:teuthology.orchestra.run.smithi159.stdout: 5 270661 10.13 MB/sec execute 520 sec latency 149.477 ms 2024-03-30T13:34:18.936 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:18 smithi187 ceph-mon[53604]: pgmap v205: 97 pgs: 7 peering, 12 stale+active+clean, 78 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.9 MiB/s wr, 416 op/s 2024-03-30T13:34:18.936 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 podman[58315]: 2024-03-30 13:34:18.563976097 +0000 UTC m=+0.021851061 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:34:18.936 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 podman[58315]: 2024-03-30 13:34:18.690367393 +0000 UTC m=+0.148242323 container create 4b825344dc6dcda195aeadf544dd7c0e383e6a554287322ee98f622aa7cf0947 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-30T13:34:18.936 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 podman[58315]: 2024-03-30 13:34:18.935151038 +0000 UTC m=+0.393025983 container init 4b825344dc6dcda195aeadf544dd7c0e383e6a554287322ee98f622aa7cf0947 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-30T13:34:19.272 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:18 smithi187 podman[58315]: 2024-03-30 13:34:18.939623086 +0000 UTC m=+0.397498025 container start 4b825344dc6dcda195aeadf544dd7c0e383e6a554287322ee98f622aa7cf0947 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD) 2024-03-30T13:34:19.272 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 podman[58315]: 2024-03-30 13:34:19.035317035 +0000 UTC m=+0.493191975 container attach 4b825344dc6dcda195aeadf544dd7c0e383e6a554287322ee98f622aa7cf0947 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate, org.label-schema.build-date=20240326, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-30T13:34:19.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-mon[53604]: pgmap v206: 97 pgs: 6 active+undersized, 23 active+undersized+degraded, 7 peering, 3 stale+active+clean, 58 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 424 op/s; 211/1635 objects degraded (12.905%) 2024-03-30T13:34:19.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-mon[53604]: Health check failed: Degraded data redundancy: 211/1635 objects degraded (12.905%), 23 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:19.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate[58330]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-30T13:34:19.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate[58330]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-30T13:34:19.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 bash[58315]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-30T13:34:19.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 bash[58315]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-30T13:34:19.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 bash[58315]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-30T13:34:19.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 bash[58315]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-30T13:34:19.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 bash[58315]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-30T13:34:19.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 bash[58315]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-30T13:34:19.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 bash[58315]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-30T13:34:19.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate[58330]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-30T13:34:19.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate[58330]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-30T13:34:19.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate[58330]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-30T13:34:19.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate[58330]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-30T13:34:19.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate[58330]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-30T13:34:19.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:19 smithi187 podman[58452]: 2024-03-30 13:34:19.522437622 +0000 UTC m=+0.022636880 container died 4b825344dc6dcda195aeadf544dd7c0e383e6a554287322ee98f622aa7cf0947 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:34:19.793 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:19 smithi159 ceph-mon[67002]: pgmap v206: 97 pgs: 6 active+undersized, 23 active+undersized+degraded, 7 peering, 3 stale+active+clean, 58 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 424 op/s; 211/1635 objects degraded (12.905%) 2024-03-30T13:34:19.793 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:19 smithi159 ceph-mon[67002]: Health check failed: Degraded data redundancy: 211/1635 objects degraded (12.905%), 23 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:19.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 270941 10.12 MB/sec execute 521 sec latency 42.006 ms 2024-03-30T13:34:20.522 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:20 smithi187 podman[58452]: 2024-03-30 13:34:20.224281566 +0000 UTC m=+0.724480821 container remove 4b825344dc6dcda195aeadf544dd7c0e383e6a554287322ee98f622aa7cf0947 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-activate, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:34:20.522 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:20 smithi187 podman[58512]: 2024-03-30 13:34:20.35175923 +0000 UTC m=+0.020832811 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:34:20.522 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:20 smithi187 podman[58512]: 2024-03-30 13:34:20.467204506 +0000 UTC m=+0.136278091 container create 9611d3292acb167c37b5ac7f4d00467c9dc2560d899414d2b5c66720e1756739 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-30T13:34:20.934 INFO:teuthology.orchestra.run.smithi159.stdout: 5 271238 10.11 MB/sec execute 522 sec latency 46.450 ms 2024-03-30T13:34:21.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:20 smithi187 podman[58512]: 2024-03-30 13:34:20.686862443 +0000 UTC m=+0.355936037 container init 9611d3292acb167c37b5ac7f4d00467c9dc2560d899414d2b5c66720e1756739 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-03-30T13:34:21.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:20 smithi187 podman[58512]: 2024-03-30 13:34:20.691295724 +0000 UTC m=+0.360369306 container start 9611d3292acb167c37b5ac7f4d00467c9dc2560d899414d2b5c66720e1756739 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-30T13:34:21.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:20 smithi187 ceph-osd[58530]: -- 172.21.15.187:0/1077005308 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e7e65d9680 con 0x55e7e65a3000 2024-03-30T13:34:21.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:20 smithi187 bash[58512]: 9611d3292acb167c37b5ac7f4d00467c9dc2560d899414d2b5c66720e1756739 2024-03-30T13:34:21.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:20 smithi187 systemd[1]: Started Ceph osd.4 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:34:21.522 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:21 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:34:21.231+0000 7fde785ec7c0 -1 Falling back to public interface 2024-03-30T13:34:21.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:21.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:21.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:34:21.862 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:21 smithi159 ceph-mon[67002]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 502 op/s; 266/1647 objects degraded (16.151%) 2024-03-30T13:34:21.933 INFO:teuthology.orchestra.run.smithi159.stdout: 5 271558 10.12 MB/sec execute 523 sec latency 51.471 ms 2024-03-30T13:34:22.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:22.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:22.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:34:22.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:21 smithi187 ceph-mon[53604]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 502 op/s; 266/1647 objects degraded (16.151%) 2024-03-30T13:34:22.934 INFO:teuthology.orchestra.run.smithi159.stdout: 5 271920 10.12 MB/sec execute 524 sec latency 60.132 ms 2024-03-30T13:34:23.934 INFO:teuthology.orchestra.run.smithi159.stdout: 5 272269 10.11 MB/sec execute 525 sec latency 62.238 ms 2024-03-30T13:34:24.473 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:24 smithi159 ceph-mon[67002]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.2 MiB/s wr, 290 op/s; 273/1680 objects degraded (16.250%) 2024-03-30T13:34:24.647 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:24 smithi187 ceph-mon[53604]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.2 MiB/s wr, 290 op/s; 273/1680 objects degraded (16.250%) 2024-03-30T13:34:24.934 INFO:teuthology.orchestra.run.smithi159.stdout: 5 272563 10.10 MB/sec execute 526 sec latency 45.033 ms 2024-03-30T13:34:25.934 INFO:teuthology.orchestra.run.smithi159.stdout: 5 272963 10.10 MB/sec execute 527 sec latency 41.367 ms 2024-03-30T13:34:26.011 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:26.011 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:26.011 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:25 smithi159 ceph-mon[67002]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.9 MiB/s wr, 416 op/s; 263/1641 objects degraded (16.027%) 2024-03-30T13:34:26.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:26.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:26.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:25 smithi187 ceph-mon[53604]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.9 MiB/s wr, 416 op/s; 263/1641 objects degraded (16.027%) 2024-03-30T13:34:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: 5 273461 10.12 MB/sec execute 528 sec latency 31.303 ms 2024-03-30T13:34:27.043 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:26 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 263/1641 objects degraded (16.027%), 28 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:27.043 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:27.043 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:26 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 263/1641 objects degraded (16.027%), 28 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:26 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:27.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:26 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:27.936 INFO:teuthology.orchestra.run.smithi159.stdout: 5 273901 10.13 MB/sec execute 529 sec latency 45.875 ms 2024-03-30T13:34:28.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:28 smithi159 ceph-mon[67002]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.8 MiB/s wr, 385 op/s; 264/1638 objects degraded (16.117%) 2024-03-30T13:34:28.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:28.602 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:34:28.732 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:28 smithi187 ceph-mon[53604]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.8 MiB/s wr, 385 op/s; 264/1638 objects degraded (16.117%) 2024-03-30T13:34:28.732 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:28.732 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:34:28.936 INFO:teuthology.orchestra.run.smithi159.stdout: 5 274144 10.12 MB/sec execute 530 sec latency 66.030 ms 2024-03-30T13:34:29.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:34:28.818+0000 7fde785ec7c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-03-30T13:34:29.936 INFO:teuthology.orchestra.run.smithi159.stdout: 5 274382 10.10 MB/sec execute 531 sec latency 56.830 ms 2024-03-30T13:34:30.649 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 525 op/s; 280/1707 objects degraded (16.403%) 2024-03-30T13:34:30.649 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:30.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:30.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:34:30.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:34:30.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:30.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: from='osd.4 [v2:172.21.15.187:6800/908431046,v1:172.21.15.187:6801/908431046]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-30T13:34:30.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-30T13:34:30.650 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:30 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:30.771 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:34:30.401+0000 7fde785ec7c0 -1 osd.4 113 log_to_monitors true 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 525 op/s; 280/1707 objects degraded (16.403%) 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: from='osd.4 [v2:172.21.15.187:6800/908431046,v1:172.21.15.187:6801/908431046]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-30T13:34:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:30 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:30.936 INFO:teuthology.orchestra.run.smithi159.stdout: 5 274662 10.10 MB/sec execute 532 sec latency 41.643 ms 2024-03-30T13:34:31.675 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: pgmap v212: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 514 op/s; 279/1737 objects degraded (16.062%) 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 279/1737 objects degraded (16.062%), 28 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: osdmap e116: 8 total, 7 up, 8 in 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='osd.4 [v2:172.21.15.187:6800/908431046,v1:172.21.15.187:6801/908431046]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:34:31.676 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:31 smithi159 ceph-mon[67002]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:34:31.886 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: pgmap v212: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 514 op/s; 279/1737 objects degraded (16.062%) 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 279/1737 objects degraded (16.062%), 28 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: osdmap e116: 8 total, 7 up, 8 in 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='osd.4 [v2:172.21.15.187:6800/908431046,v1:172.21.15.187:6801/908431046]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:34:31.887 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-mon[53604]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:34:31.936 INFO:teuthology.orchestra.run.smithi159.stdout: 5 274937 10.09 MB/sec execute 533 sec latency 93.018 ms 2024-03-30T13:34:32.272 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:34:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:34:31.885+0000 7fde65a3c700 -1 osd.4 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:34:32.700 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:32 smithi159 ceph-mon[67002]: failed to encode map e116 with expected crc 2024-03-30T13:34:32.700 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:32 smithi159 ceph-mon[67002]: failed to encode map e116 with expected crc 2024-03-30T13:34:32.700 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:32 smithi159 ceph-mon[67002]: failed to encode map e116 with expected crc 2024-03-30T13:34:32.700 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:32 smithi159 ceph-mon[67002]: failed to encode map e116 with expected crc 2024-03-30T13:34:32.701 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:32 smithi159 ceph-mon[67002]: from='osd.4 ' entity='osd.4' 2024-03-30T13:34:32.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 275271 10.10 MB/sec execute 534 sec latency 42.259 ms 2024-03-30T13:34:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:32 smithi187 ceph-mon[53604]: failed to encode map e116 with expected crc 2024-03-30T13:34:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:32 smithi187 ceph-mon[53604]: failed to encode map e116 with expected crc 2024-03-30T13:34:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:32 smithi187 ceph-mon[53604]: failed to encode map e116 with expected crc 2024-03-30T13:34:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:32 smithi187 ceph-mon[53604]: failed to encode map e116 with expected crc 2024-03-30T13:34:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:32 smithi187 ceph-mon[53604]: from='osd.4 ' entity='osd.4' 2024-03-30T13:34:33.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 275282 10.08 MB/sec execute 535 sec latency 1036.513 ms 2024-03-30T13:34:34.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:33 smithi159 ceph-mon[67002]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 535 op/s; 271/1719 objects degraded (15.765%) 2024-03-30T13:34:34.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:33 smithi159 ceph-mon[67002]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:34:34.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:33 smithi159 ceph-mon[67002]: osd.4 [v2:172.21.15.187:6800/908431046,v1:172.21.15.187:6801/908431046] boot 2024-03-30T13:34:34.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:33 smithi159 ceph-mon[67002]: osdmap e117: 8 total, 8 up, 8 in 2024-03-30T13:34:34.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:33 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:34:34.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:33 smithi187 ceph-mon[53604]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 535 op/s; 271/1719 objects degraded (15.765%) 2024-03-30T13:34:34.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:33 smithi187 ceph-mon[53604]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:34:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:33 smithi187 ceph-mon[53604]: osd.4 [v2:172.21.15.187:6800/908431046,v1:172.21.15.187:6801/908431046] boot 2024-03-30T13:34:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:33 smithi187 ceph-mon[53604]: osdmap e117: 8 total, 8 up, 8 in 2024-03-30T13:34:34.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:33 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-30T13:34:34.938 INFO:teuthology.orchestra.run.smithi159.stdout: 5 275356 10.06 MB/sec execute 536 sec latency 2003.196 ms 2024-03-30T13:34:34.972 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:34 smithi159 ceph-mon[67002]: failed to encode map e117 with expected crc 2024-03-30T13:34:34.972 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:34 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:34.972 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:34 smithi159 ceph-mon[67002]: osdmap e118: 8 total, 8 up, 8 in 2024-03-30T13:34:34.973 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:34 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:34.973 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:34 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:34.973 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:34 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:34 smithi187 ceph-mon[53604]: failed to encode map e117 with expected crc 2024-03-30T13:34:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:34 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:34 smithi187 ceph-mon[53604]: osdmap e118: 8 total, 8 up, 8 in 2024-03-30T13:34:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:34 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:34 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:34 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.748 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.748 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.748 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.749 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.749 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.749 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.749 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: failed to encode map e118 with expected crc 2024-03-30T13:34:35.749 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: pgmap v217: 97 pgs: 14 peering, 13 active+undersized, 14 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.2 MiB/s wr, 246 op/s; 136/1710 objects degraded (7.953%) 2024-03-30T13:34:35.749 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: osdmap e119: 8 total, 8 up, 8 in 2024-03-30T13:34:35.749 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:35 smithi159 ceph-mon[67002]: failed to encode map e119 with expected crc 2024-03-30T13:34:35.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 275706 10.07 MB/sec execute 537 sec latency 2055.526 ms 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: failed to encode map e118 with expected crc 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: pgmap v217: 97 pgs: 14 peering, 13 active+undersized, 14 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.2 MiB/s wr, 246 op/s; 136/1710 objects degraded (7.953%) 2024-03-30T13:34:36.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: osdmap e119: 8 total, 8 up, 8 in 2024-03-30T13:34:36.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:35 smithi187 ceph-mon[53604]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.774 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:36 smithi159 ceph-mon[67002]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.774 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:36 smithi159 ceph-mon[67002]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.775 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:36 smithi159 ceph-mon[67002]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.775 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:36 smithi159 ceph-mon[67002]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.775 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:36 smithi159 ceph-mon[67002]: failed to encode map e120 with expected crc 2024-03-30T13:34:36.775 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:36 smithi159 ceph-mon[67002]: osdmap e120: 8 total, 8 up, 8 in 2024-03-30T13:34:36.775 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:36 smithi159 ceph-mon[67002]: failed to encode map e120 with expected crc 2024-03-30T13:34:36.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:36 smithi187 ceph-mon[53604]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:36 smithi187 ceph-mon[53604]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:36 smithi187 ceph-mon[53604]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:36 smithi187 ceph-mon[53604]: failed to encode map e119 with expected crc 2024-03-30T13:34:36.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:36 smithi187 ceph-mon[53604]: failed to encode map e120 with expected crc 2024-03-30T13:34:36.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:36 smithi187 ceph-mon[53604]: osdmap e120: 8 total, 8 up, 8 in 2024-03-30T13:34:36.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:36 smithi187 ceph-mon[53604]: failed to encode map e120 with expected crc 2024-03-30T13:34:36.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 276000 10.05 MB/sec execute 538 sec latency 52.664 ms 2024-03-30T13:34:37.796 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:37 smithi159 ceph-mon[67002]: failed to encode map e120 with expected crc 2024-03-30T13:34:37.797 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:37 smithi159 ceph-mon[67002]: failed to encode map e120 with expected crc 2024-03-30T13:34:37.797 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:37 smithi159 ceph-mon[67002]: failed to encode map e120 with expected crc 2024-03-30T13:34:37.797 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:37 smithi159 ceph-mon[67002]: pgmap v220: 97 pgs: 1 active+recovery_wait+degraded, 14 peering, 10 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.4 MiB/s wr, 425 op/s; 144/1677 objects degraded (8.587%); 22 KiB/s, 80 keys/s, 1 objects/s recovering 2024-03-30T13:34:37.797 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:37 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 144/1677 objects degraded (8.587%), 12 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:37.797 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:37 smithi159 ceph-mon[67002]: osdmap e121: 8 total, 8 up, 8 in 2024-03-30T13:34:37.797 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:37 smithi159 ceph-mon[67002]: failed to encode map e121 with expected crc 2024-03-30T13:34:37.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 276282 10.04 MB/sec execute 539 sec latency 669.836 ms 2024-03-30T13:34:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:37 smithi187 ceph-mon[53604]: failed to encode map e120 with expected crc 2024-03-30T13:34:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:37 smithi187 ceph-mon[53604]: failed to encode map e120 with expected crc 2024-03-30T13:34:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:37 smithi187 ceph-mon[53604]: failed to encode map e120 with expected crc 2024-03-30T13:34:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:37 smithi187 ceph-mon[53604]: pgmap v220: 97 pgs: 1 active+recovery_wait+degraded, 14 peering, 10 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.4 MiB/s wr, 425 op/s; 144/1677 objects degraded (8.587%); 22 KiB/s, 80 keys/s, 1 objects/s recovering 2024-03-30T13:34:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:37 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 144/1677 objects degraded (8.587%), 12 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:37 smithi187 ceph-mon[53604]: osdmap e121: 8 total, 8 up, 8 in 2024-03-30T13:34:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:37 smithi187 ceph-mon[53604]: failed to encode map e121 with expected crc 2024-03-30T13:34:38.808 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:38 smithi159 ceph-mon[67002]: failed to encode map e121 with expected crc 2024-03-30T13:34:38.808 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:38 smithi159 ceph-mon[67002]: failed to encode map e121 with expected crc 2024-03-30T13:34:38.808 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:38 smithi159 ceph-mon[67002]: osdmap e122: 8 total, 8 up, 8 in 2024-03-30T13:34:38.808 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:38 smithi159 ceph-mon[67002]: failed to encode map e122 with expected crc 2024-03-30T13:34:38.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 276507 10.03 MB/sec execute 540 sec latency 704.931 ms 2024-03-30T13:34:39.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:38 smithi187 ceph-mon[53604]: failed to encode map e121 with expected crc 2024-03-30T13:34:39.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:38 smithi187 ceph-mon[53604]: failed to encode map e121 with expected crc 2024-03-30T13:34:39.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:38 smithi187 ceph-mon[53604]: osdmap e122: 8 total, 8 up, 8 in 2024-03-30T13:34:39.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:38 smithi187 ceph-mon[53604]: failed to encode map e122 with expected crc 2024-03-30T13:34:39.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:39 smithi159 ceph-mon[67002]: failed to encode map e122 with expected crc 2024-03-30T13:34:39.598 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:39 smithi159 ceph-mon[67002]: failed to encode map e122 with expected crc 2024-03-30T13:34:39.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:39 smithi159 ceph-mon[67002]: failed to encode map e122 with expected crc 2024-03-30T13:34:39.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:39 smithi159 ceph-mon[67002]: pgmap v223: 97 pgs: 2 active+recovery_wait+degraded, 15 peering, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.3 MiB/s wr, 356 op/s; 79/1665 objects degraded (4.745%); 277 KiB/s, 80 keys/s, 24 objects/s recovering 2024-03-30T13:34:39.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:39 smithi159 ceph-mon[67002]: osdmap e123: 8 total, 8 up, 8 in 2024-03-30T13:34:39.599 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:39 smithi159 ceph-mon[67002]: failed to encode map e123 with expected crc 2024-03-30T13:34:39.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 276737 10.03 MB/sec execute 541 sec latency 869.650 ms 2024-03-30T13:34:40.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:39 smithi187 ceph-mon[53604]: failed to encode map e122 with expected crc 2024-03-30T13:34:40.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:39 smithi187 ceph-mon[53604]: failed to encode map e122 with expected crc 2024-03-30T13:34:40.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:39 smithi187 ceph-mon[53604]: failed to encode map e122 with expected crc 2024-03-30T13:34:40.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:39 smithi187 ceph-mon[53604]: pgmap v223: 97 pgs: 2 active+recovery_wait+degraded, 15 peering, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.3 MiB/s wr, 356 op/s; 79/1665 objects degraded (4.745%); 277 KiB/s, 80 keys/s, 24 objects/s recovering 2024-03-30T13:34:40.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:39 smithi187 ceph-mon[53604]: osdmap e123: 8 total, 8 up, 8 in 2024-03-30T13:34:40.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:39 smithi187 ceph-mon[53604]: failed to encode map e123 with expected crc 2024-03-30T13:34:40.640 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:40 smithi159 ceph-mon[67002]: failed to encode map e123 with expected crc 2024-03-30T13:34:40.640 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:40 smithi159 ceph-mon[67002]: failed to encode map e123 with expected crc 2024-03-30T13:34:40.641 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:40 smithi159 ceph-mon[67002]: failed to encode map e123 with expected crc 2024-03-30T13:34:40.641 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:40 smithi159 ceph-mon[67002]: failed to encode map e123 with expected crc 2024-03-30T13:34:40.641 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:40 smithi159 ceph-mon[67002]: osdmap e124: 8 total, 8 up, 8 in 2024-03-30T13:34:40.641 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:40 smithi159 ceph-mon[67002]: failed to encode map e124 with expected crc 2024-03-30T13:34:40.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 277087 10.04 MB/sec execute 542 sec latency 915.032 ms 2024-03-30T13:34:41.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:40 smithi187 ceph-mon[53604]: failed to encode map e123 with expected crc 2024-03-30T13:34:41.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:40 smithi187 ceph-mon[53604]: failed to encode map e123 with expected crc 2024-03-30T13:34:41.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:40 smithi187 ceph-mon[53604]: failed to encode map e123 with expected crc 2024-03-30T13:34:41.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:40 smithi187 ceph-mon[53604]: failed to encode map e123 with expected crc 2024-03-30T13:34:41.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:40 smithi187 ceph-mon[53604]: osdmap e124: 8 total, 8 up, 8 in 2024-03-30T13:34:41.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:40 smithi187 ceph-mon[53604]: failed to encode map e124 with expected crc 2024-03-30T13:34:41.648 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:41 smithi159 ceph-mon[67002]: failed to encode map e124 with expected crc 2024-03-30T13:34:41.648 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:41 smithi159 ceph-mon[67002]: failed to encode map e124 with expected crc 2024-03-30T13:34:41.648 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:41 smithi159 ceph-mon[67002]: pgmap v226: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.5 MiB/s wr, 456 op/s; 67/1638 objects degraded (4.090%); 400 KiB/s, 66 objects/s recovering 2024-03-30T13:34:41.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 277444 10.04 MB/sec execute 543 sec latency 38.832 ms 2024-03-30T13:34:42.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:41 smithi187 ceph-mon[53604]: failed to encode map e124 with expected crc 2024-03-30T13:34:42.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:41 smithi187 ceph-mon[53604]: failed to encode map e124 with expected crc 2024-03-30T13:34:42.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:41 smithi187 ceph-mon[53604]: pgmap v226: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.5 MiB/s wr, 456 op/s; 67/1638 objects degraded (4.090%); 400 KiB/s, 66 objects/s recovering 2024-03-30T13:34:42.653 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:42 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 67/1638 objects degraded (4.090%), 5 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:42.653 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:42.653 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:34:42.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 277790 10.03 MB/sec execute 544 sec latency 40.682 ms 2024-03-30T13:34:43.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:42 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 67/1638 objects degraded (4.090%), 5 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:43.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:43.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:34:43.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:43 smithi159 ceph-mon[67002]: pgmap v227: 97 pgs: 1 active+recovery_wait+degraded, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.2 MiB/s wr, 362 op/s; 30/1674 objects degraded (1.792%); 1.2 MiB/s, 45 objects/s recovering 2024-03-30T13:34:43.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 278005 10.02 MB/sec execute 545 sec latency 41.692 ms 2024-03-30T13:34:44.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:43 smithi187 ceph-mon[53604]: pgmap v227: 97 pgs: 1 active+recovery_wait+degraded, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.2 MiB/s wr, 362 op/s; 30/1674 objects degraded (1.792%); 1.2 MiB/s, 45 objects/s recovering 2024-03-30T13:34:44.743 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:44 smithi159 ceph-mon[67002]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1674 objects degraded (1.792%), 3 pgs degraded) 2024-03-30T13:34:44.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 278303 10.02 MB/sec execute 546 sec latency 39.676 ms 2024-03-30T13:34:45.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:44 smithi187 ceph-mon[53604]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1674 objects degraded (1.792%), 3 pgs degraded) 2024-03-30T13:34:45.772 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:45 smithi159 ceph-mon[67002]: pgmap v228: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.9 MiB/s wr, 359 op/s; 1015 KiB/s, 37 objects/s recovering 2024-03-30T13:34:45.772 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:45 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-30T13:34:45.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 278566 10.01 MB/sec execute 547 sec latency 53.996 ms 2024-03-30T13:34:46.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:45 smithi187 ceph-mon[53604]: pgmap v228: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.9 MiB/s wr, 359 op/s; 1015 KiB/s, 37 objects/s recovering 2024-03-30T13:34:46.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:45 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-30T13:34:46.463 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:34:46.791 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:46 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-30T13:34:46.791 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:46 smithi159 ceph-mon[67002]: Upgrade: osd.5 is safe to restart 2024-03-30T13:34:46.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 278807 10.01 MB/sec execute 548 sec latency 57.438 ms 2024-03-30T13:34:47.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:46 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-30T13:34:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:46 smithi187 ceph-mon[53604]: Upgrade: osd.5 is safe to restart 2024-03-30T13:34:47.325 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (8m) 60s ago 17m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (4m) 60s ago 17m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (4m) 22s ago 16m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 60s ago 17m - - 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (11m) 60s ago 11m 38.5M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (11m) 22s ago 11m 17.6M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (7m) 60s ago 18m 507M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (6m) 22s ago 16m 451M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (5m) 60s ago 18m 49.7M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (5m) 22s ago 16m 40.5M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (11m) 60s ago 11m 283M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:34:47.326 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (11m) 22s ago 11m 86.5M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (7m) 60s ago 17m 13.0M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (7m) 22s ago 16m 19.8M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (3m) 60s ago 15m 356M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (2m) 60s ago 15m 277M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (117s) 60s ago 15m 241M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (68s) 60s ago 14m 12.0M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (26s) 22s ago 14m 14.8M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (13m) 22s ago 13m 668M 2704M 17.2.0 e1d6a67b021e 083e4057cfd3 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (13m) 22s ago 13m 582M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (13m) 22s ago 13m 591M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:34:47.327 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (6m) 60s ago 16m 63.2M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:34:47.820 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:47 smithi159 ceph-mon[67002]: pgmap v229: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.4 MiB/s wr, 479 op/s; 819 KiB/s, 30 objects/s recovering 2024-03-30T13:34:47.820 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:47 smithi159 ceph-mon[67002]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:47.821 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:47.821 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-30T13:34:47.821 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 5 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:34:47.844 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:34:47.845 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:34:47.845 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:34:47.845 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-30T13:34:47.845 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 9 2024-03-30T13:34:47.845 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:34:47.845 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:34:47.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 279087 10.01 MB/sec execute 549 sec latency 53.977 ms 2024-03-30T13:34:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:47 smithi187 ceph-mon[53604]: pgmap v229: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.4 MiB/s wr, 479 op/s; 819 KiB/s, 30 objects/s recovering 2024-03-30T13:34:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:47 smithi187 ceph-mon[53604]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-30T13:34:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:34:48.293 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:34:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "11/23 daemons upgraded", 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:34:48.294 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:34:48.847 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:48 smithi159 ceph-mon[67002]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:48.847 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:48 smithi159 ceph-mon[67002]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:48.847 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:48 smithi159 ceph-mon[67002]: Upgrade: Updating osd.5 2024-03-30T13:34:48.848 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:48 smithi159 ceph-mon[67002]: Deploying daemon osd.5 on smithi187 2024-03-30T13:34:48.848 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:48 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/2050071282' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:48.924 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:34:48.924 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:34:48.924 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:34:48.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 279339 10.00 MB/sec execute 550 sec latency 59.063 ms 2024-03-30T13:34:49.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:48 smithi187 ceph-mon[53604]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:48 smithi187 ceph-mon[53604]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:48 smithi187 ceph-mon[53604]: Upgrade: Updating osd.5 2024-03-30T13:34:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:48 smithi187 ceph-mon[53604]: Deploying daemon osd.5 on smithi187 2024-03-30T13:34:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:48 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/2050071282' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:34:49.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:49 smithi187 ceph-mon[53604]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:49.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:49 smithi187 ceph-mon[53604]: pgmap v230: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.9 MiB/s wr, 401 op/s; 694 KiB/s, 8 objects/s recovering 2024-03-30T13:34:49.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:49 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/4209258471' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:34:49.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:49 smithi159 ceph-mon[67002]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:34:49.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:49 smithi159 ceph-mon[67002]: pgmap v230: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.9 MiB/s wr, 401 op/s; 694 KiB/s, 8 objects/s recovering 2024-03-30T13:34:49.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:49 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/4209258471' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:34:49.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 279680 9.99 MB/sec execute 551 sec latency 42.075 ms 2024-03-30T13:34:50.876 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:50 smithi159 ceph-mon[67002]: pgmap v231: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 572 KiB/s, 6 objects/s recovering 2024-03-30T13:34:50.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 280015 9.99 MB/sec execute 552 sec latency 91.895 ms 2024-03-30T13:34:50.946 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:50 smithi187 ceph-mon[53604]: pgmap v231: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 572 KiB/s, 6 objects/s recovering 2024-03-30T13:34:51.247 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:50 smithi187 systemd[1]: Stopping Ceph osd.5 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:34:51.522 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:51 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[34744]: 2024-03-30T13:34:51.271+0000 7f002c421700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:34:51.522 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:51 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[34744]: 2024-03-30T13:34:51.271+0000 7f002c421700 -1 osd.5 124 *** Got signal Terminated *** 2024-03-30T13:34:51.522 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:51 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[34744]: 2024-03-30T13:34:51.271+0000 7f002c421700 -1 osd.5 124 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:34:51.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 280287 9.99 MB/sec execute 553 sec latency 256.142 ms 2024-03-30T13:34:52.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:51 smithi159 ceph-mon[67002]: osd.5 marked itself down and dead 2024-03-30T13:34:52.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:51 smithi187 ceph-mon[53604]: osd.5 marked itself down and dead 2024-03-30T13:34:52.522 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:52 smithi187 podman[60324]: 2024-03-30 13:34:52.348799775 +0000 UTC m=+1.192338010 container died 083e4057cfd3cdb4337f1bbfcaa8b8b7707dad11043dba2f9b61fb59e0720630 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-type=git, architecture=x86_64, ceph=True, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, name=centos-stream, distribution-scope=public, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-30T13:34:52.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 280362 9.98 MB/sec execute 554 sec latency 1059.934 ms 2024-03-30T13:34:52.941 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:52 smithi159 ceph-mon[67002]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:34:52.941 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:52 smithi159 ceph-mon[67002]: failed to encode map e125 with expected crc 2024-03-30T13:34:52.941 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:52 smithi159 ceph-mon[67002]: osdmap e125: 8 total, 7 up, 8 in 2024-03-30T13:34:52.941 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:52 smithi159 ceph-mon[67002]: pgmap v233: 97 pgs: 12 stale+active+clean, 85 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.3 MiB/s wr, 390 op/s; 48 KiB/s, 2 objects/s recovering 2024-03-30T13:34:52.941 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:52 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:52 smithi187 ceph-mon[53604]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:34:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:52 smithi187 ceph-mon[53604]: failed to encode map e125 with expected crc 2024-03-30T13:34:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:52 smithi187 ceph-mon[53604]: osdmap e125: 8 total, 7 up, 8 in 2024-03-30T13:34:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:52 smithi187 ceph-mon[53604]: pgmap v233: 97 pgs: 12 stale+active+clean, 85 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.3 MiB/s wr, 390 op/s; 48 KiB/s, 2 objects/s recovering 2024-03-30T13:34:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:52 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:53.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:52 smithi187 podman[60324]: 2024-03-30 13:34:52.893070954 +0000 UTC m=+1.736609174 container cleanup 083e4057cfd3cdb4337f1bbfcaa8b8b7707dad11043dba2f9b61fb59e0720630 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, release=754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=HEAD) 2024-03-30T13:34:53.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:52 smithi187 bash[60324]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5 2024-03-30T13:34:53.522 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:53 smithi187 podman[60353]: 2024-03-30 13:34:53.202848062 +0000 UTC m=+0.857090928 container remove 083e4057cfd3cdb4337f1bbfcaa8b8b7707dad11043dba2f9b61fb59e0720630 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, ceph=True, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_CLEAN=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8) 2024-03-30T13:34:53.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:53 smithi159 ceph-mon[67002]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:53 smithi159 ceph-mon[67002]: osdmap e126: 8 total, 7 up, 8 in 2024-03-30T13:34:53.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:53 smithi159 ceph-mon[67002]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.723 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:53 smithi159 ceph-mon[67002]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.724 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:53 smithi159 ceph-mon[67002]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.724 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:53 smithi159 ceph-mon[67002]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.724 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:53 smithi159 ceph-mon[67002]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.724 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:53 smithi159 ceph-mon[67002]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.858 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:53 smithi187 ceph-mon[53604]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.858 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:53 smithi187 ceph-mon[53604]: osdmap e126: 8 total, 7 up, 8 in 2024-03-30T13:34:53.858 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:53 smithi187 ceph-mon[53604]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.858 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:53 smithi187 ceph-mon[53604]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.858 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:53 smithi187 ceph-mon[53604]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.858 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:53 smithi187 ceph-mon[53604]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.858 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:53 smithi187 ceph-mon[53604]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.858 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:53 smithi187 ceph-mon[53604]: failed to encode map e126 with expected crc 2024-03-30T13:34:53.859 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:53 smithi187 podman[60441]: 2024-03-30 13:34:53.483944955 +0000 UTC m=+0.020573737 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:34:53.859 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:53 smithi187 podman[60441]: 2024-03-30 13:34:53.603915954 +0000 UTC m=+0.140544735 container create 34963cc64d758535911255ee5df22891f425c33e26bb84a915797ccb983f6e62 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-30T13:34:53.859 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:53 smithi187 podman[60441]: 2024-03-30 13:34:53.857187509 +0000 UTC m=+0.393816284 container init 34963cc64d758535911255ee5df22891f425c33e26bb84a915797ccb983f6e62 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:34:53.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 280761 9.99 MB/sec execute 555 sec latency 59.401 ms 2024-03-30T13:34:54.221 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:53 smithi187 podman[60441]: 2024-03-30 13:34:53.862443609 +0000 UTC m=+0.399072391 container start 34963cc64d758535911255ee5df22891f425c33e26bb84a915797ccb983f6e62 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:34:54.221 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:53 smithi187 podman[60441]: 2024-03-30 13:34:53.965340593 +0000 UTC m=+0.501969388 container attach 34963cc64d758535911255ee5df22891f425c33e26bb84a915797ccb983f6e62 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5) 2024-03-30T13:34:54.521 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:54 smithi187 podman[60441]: 2024-03-30 13:34:54.220453568 +0000 UTC m=+0.757082343 container died 34963cc64d758535911255ee5df22891f425c33e26bb84a915797ccb983f6e62 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-30T13:34:54.705 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:54 smithi159 ceph-mon[67002]: pgmap v235: 97 pgs: 4 active+undersized, 10 active+undersized+degraded, 3 peering, 7 stale+active+clean, 73 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 424 op/s; 66/1632 objects degraded (4.044%) 2024-03-30T13:34:54.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 281149 9.98 MB/sec execute 556 sec latency 48.262 ms 2024-03-30T13:34:55.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:54 smithi187 ceph-mon[53604]: pgmap v235: 97 pgs: 4 active+undersized, 10 active+undersized+degraded, 3 peering, 7 stale+active+clean, 73 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 424 op/s; 66/1632 objects degraded (4.044%) 2024-03-30T13:34:55.384 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:55 smithi187 podman[60466]: 2024-03-30 13:34:55.12892294 +0000 UTC m=+0.901865641 container remove 34963cc64d758535911255ee5df22891f425c33e26bb84a915797ccb983f6e62 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=) 2024-03-30T13:34:55.385 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:55 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.5.service: Deactivated successfully. 2024-03-30T13:34:55.385 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:55 smithi187 systemd[1]: Stopped Ceph osd.5 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:34:55.385 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:55 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.5.service: Consumed 6min 4.018s CPU time. 2024-03-30T13:34:55.647 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:55 smithi187 systemd[1]: Starting Ceph osd.5 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:34:55.647 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:55 smithi187 podman[60566]: 2024-03-30 13:34:55.525687668 +0000 UTC m=+0.020395979 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:34:55.734 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:55 smithi159 ceph-mon[67002]: Health check failed: Degraded data redundancy: 66/1632 objects degraded (4.044%), 10 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:55.938 INFO:teuthology.orchestra.run.smithi159.stdout: 5 281436 9.98 MB/sec execute 557 sec latency 42.463 ms 2024-03-30T13:34:56.018 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:55 smithi187 ceph-mon[53604]: Health check failed: Degraded data redundancy: 66/1632 objects degraded (4.044%), 10 pgs degraded (PG_DEGRADED) 2024-03-30T13:34:56.018 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:55 smithi187 podman[60566]: 2024-03-30 13:34:55.646963141 +0000 UTC m=+0.141671434 container create 8c32b1f185342c7a34fef8bcda111c6b892e76ddfa1e9c5b463cc88dc14c121a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate, RELEASE=squid-4e50bbc, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-03-30T13:34:56.272 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 podman[60566]: 2024-03-30 13:34:56.016882747 +0000 UTC m=+0.511591032 container init 8c32b1f185342c7a34fef8bcda111c6b892e76ddfa1e9c5b463cc88dc14c121a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc) 2024-03-30T13:34:56.272 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 podman[60566]: 2024-03-30 13:34:56.020103374 +0000 UTC m=+0.514811667 container start 8c32b1f185342c7a34fef8bcda111c6b892e76ddfa1e9c5b463cc88dc14c121a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:34:56.272 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 podman[60566]: 2024-03-30 13:34:56.119119529 +0000 UTC m=+0.613827822 container attach 8c32b1f185342c7a34fef8bcda111c6b892e76ddfa1e9c5b463cc88dc14c121a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux ) 2024-03-30T13:34:56.760 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:56 smithi159 ceph-mon[67002]: pgmap v236: 97 pgs: 5 active+undersized, 18 active+undersized+degraded, 3 peering, 3 stale+active+clean, 68 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.3 MiB/s wr, 431 op/s; 132/1665 objects degraded (7.928%); 8.0 KiB/s, 0 objects/s recovering 2024-03-30T13:34:56.879 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:56 smithi187 ceph-mon[53604]: pgmap v236: 97 pgs: 5 active+undersized, 18 active+undersized+degraded, 3 peering, 3 stale+active+clean, 68 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.3 MiB/s wr, 431 op/s; 132/1665 objects degraded (7.928%); 8.0 KiB/s, 0 objects/s recovering 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate[60581]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate[60581]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 bash[60566]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 bash[60566]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 bash[60566]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 bash[60566]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 bash[60566]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 bash[60566]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 bash[60566]: --> ceph-volume raw activate successful for osd ID: 5 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate[60581]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-30T13:34:56.880 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate[60581]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-30T13:34:56.881 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate[60581]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-03-30T13:34:56.881 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate[60581]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-30T13:34:56.881 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate[60581]: --> ceph-volume raw activate successful for osd ID: 5 2024-03-30T13:34:56.881 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:56 smithi187 podman[60566]: 2024-03-30 13:34:56.567418015 +0000 UTC m=+1.062126306 container died 8c32b1f185342c7a34fef8bcda111c6b892e76ddfa1e9c5b463cc88dc14c121a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-30T13:34:56.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 281685 9.97 MB/sec execute 558 sec latency 46.093 ms 2024-03-30T13:34:57.424 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:57 smithi187 podman[60698]: 2024-03-30 13:34:57.171869068 +0000 UTC m=+0.596981950 container remove 8c32b1f185342c7a34fef8bcda111c6b892e76ddfa1e9c5b463cc88dc14c121a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:34:57.424 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:57 smithi187 podman[60761]: 2024-03-30 13:34:57.299498291 +0000 UTC m=+0.019405407 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:34:57.677 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:57 smithi187 podman[60761]: 2024-03-30 13:34:57.422870282 +0000 UTC m=+0.142777370 container create e70ee05c8ec9dd161c2f9cd82fdb2634ad35838f314db146159e996746e104f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-30T13:34:57.677 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:57 smithi187 podman[60761]: 2024-03-30 13:34:57.651263641 +0000 UTC m=+0.371170753 container init e70ee05c8ec9dd161c2f9cd82fdb2634ad35838f314db146159e996746e104f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-30T13:34:57.677 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:57 smithi187 podman[60761]: 2024-03-30 13:34:57.654685689 +0000 UTC m=+0.374592778 container start e70ee05c8ec9dd161c2f9cd82fdb2634ad35838f314db146159e996746e104f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:34:57.677 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:57 smithi187 ceph-osd[60780]: -- 172.21.15.187:0/4100125113 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eba379f680 con 0x55eba3768c00 2024-03-30T13:34:57.937 INFO:teuthology.orchestra.run.smithi159.stdout: 5 281958 9.97 MB/sec execute 559 sec latency 69.015 ms 2024-03-30T13:34:58.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:57 smithi187 bash[60761]: e70ee05c8ec9dd161c2f9cd82fdb2634ad35838f314db146159e996746e104f1 2024-03-30T13:34:58.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:57 smithi187 systemd[1]: Started Ceph osd.5 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:34:58.500 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:34:58 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:34:58.203+0000 7f886451b7c0 -1 Falling back to public interface 2024-03-30T13:34:58.500 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:58.500 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:34:58.500 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:58.527 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:58.527 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:34:58.527 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:58.527 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:58.527 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:34:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:34:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:34:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 282223 9.96 MB/sec execute 560 sec latency 75.561 ms 2024-03-30T13:34:59.558 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:34:59 smithi159 ceph-mon[67002]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.9 MiB/s wr, 410 op/s; 178/1638 objects degraded (10.867%); 8.0 KiB/s, 0 objects/s recovering 2024-03-30T13:34:59.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:34:59 smithi187 ceph-mon[53604]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.9 MiB/s wr, 410 op/s; 178/1638 objects degraded (10.867%); 8.0 KiB/s, 0 objects/s recovering 2024-03-30T13:34:59.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 282540 9.96 MB/sec execute 561 sec latency 49.725 ms 2024-03-30T13:35:00.575 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:00 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 185/1698 objects degraded (10.895%), 23 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:00.712 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:00 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 185/1698 objects degraded (10.895%), 23 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:00.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 282828 9.96 MB/sec execute 562 sec latency 47.798 ms 2024-03-30T13:35:01.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:01 smithi159 ceph-mon[67002]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 400 op/s; 185/1698 objects degraded (10.895%); 7.3 KiB/s, 0 objects/s recovering 2024-03-30T13:35:01.842 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:01 smithi187 ceph-mon[53604]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 400 op/s; 185/1698 objects degraded (10.895%); 7.3 KiB/s, 0 objects/s recovering 2024-03-30T13:35:01.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 283104 9.95 MB/sec execute 563 sec latency 62.938 ms 2024-03-30T13:35:02.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 283469 9.94 MB/sec execute 564 sec latency 47.453 ms 2024-03-30T13:35:03.354 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:03.354 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:03.355 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:03 smithi159 ceph-mon[67002]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.4 MiB/s wr, 475 op/s; 187/1674 objects degraded (11.171%); 6.4 KiB/s, 0 objects/s recovering 2024-03-30T13:35:03.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:03 smithi187 ceph-mon[53604]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.4 MiB/s wr, 475 op/s; 187/1674 objects degraded (11.171%); 6.4 KiB/s, 0 objects/s recovering 2024-03-30T13:35:03.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 283703 9.93 MB/sec execute 565 sec latency 44.768 ms 2024-03-30T13:35:04.402 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:04 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:04.403 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:04 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:04.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:04 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:04.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:04 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:04.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 283957 9.94 MB/sec execute 566 sec latency 68.461 ms 2024-03-30T13:35:05.447 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:05 smithi159 ceph-mon[67002]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 427 op/s; 182/1668 objects degraded (10.911%); 5.5 KiB/s, 0 objects/s recovering 2024-03-30T13:35:05.467 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:05 smithi187 ceph-mon[53604]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 427 op/s; 182/1668 objects degraded (10.911%); 5.5 KiB/s, 0 objects/s recovering 2024-03-30T13:35:05.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:35:05 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:35:05.558+0000 7f886451b7c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-03-30T13:35:05.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 284242 9.94 MB/sec execute 567 sec latency 66.715 ms 2024-03-30T13:35:06.673 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:06 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 175/1638 objects degraded (10.684%), 23 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:06.738 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:06 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 175/1638 objects degraded (10.684%), 23 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:06.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 284498 9.93 MB/sec execute 568 sec latency 105.540 ms 2024-03-30T13:35:07.271 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:35:07.109+0000 7f886451b7c0 -1 osd.5 124 log_to_monitors true 2024-03-30T13:35:07.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 420 op/s; 175/1638 objects degraded (10.684%); 5.3 KiB/s, 0 objects/s recovering 2024-03-30T13:35:07.524 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='osd.5 [v2:172.21.15.187:6808/3777584294,v1:172.21.15.187:6809/3777584294]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:07.525 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 420 op/s; 175/1638 objects degraded (10.684%); 5.3 KiB/s, 0 objects/s recovering 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='osd.5 [v2:172.21.15.187:6808/3777584294,v1:172.21.15.187:6809/3777584294]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:07.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:07.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-30T13:35:07.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 284794 9.93 MB/sec execute 569 sec latency 50.150 ms 2024-03-30T13:35:08.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 455 op/s; 172/1626 objects degraded (10.578%) 2024-03-30T13:35:08.584 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 407 op/s; 172/1626 objects degraded (10.578%) 2024-03-30T13:35:08.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-30T13:35:08.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-30T13:35:08.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-30T13:35:08.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: failed to encode map e127 with expected crc 2024-03-30T13:35:08.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: osdmap e127: 8 total, 7 up, 8 in 2024-03-30T13:35:08.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: failed to encode map e127 with expected crc 2024-03-30T13:35:08.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: from='osd.5 [v2:172.21.15.187:6808/3777584294,v1:172.21.15.187:6809/3777584294]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:35:08.585 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:08 smithi159 ceph-mon[67002]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:35:08.939 INFO:teuthology.orchestra.run.smithi159.stdout: 5 284942 9.91 MB/sec execute 570 sec latency 129.170 ms 2024-03-30T13:35:09.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:35:08.636+0000 7f885196b700 -1 osd.5 124 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 455 op/s; 172/1626 objects degraded (10.578%) 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 407 op/s; 172/1626 objects degraded (10.578%) 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: failed to encode map e127 with expected crc 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: osdmap e127: 8 total, 7 up, 8 in 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: failed to encode map e127 with expected crc 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: from='osd.5 [v2:172.21.15.187:6808/3777584294,v1:172.21.15.187:6809/3777584294]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:35:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:08 smithi187 ceph-mon[53604]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:35:09.868 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:09 smithi159 ceph-mon[67002]: failed to encode map e127 with expected crc 2024-03-30T13:35:09.868 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:09 smithi159 ceph-mon[67002]: from='osd.5 ' entity='osd.5' 2024-03-30T13:35:09.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 285097 9.90 MB/sec execute 571 sec latency 56.613 ms 2024-03-30T13:35:10.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:09 smithi187 ceph-mon[53604]: failed to encode map e127 with expected crc 2024-03-30T13:35:10.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:09 smithi187 ceph-mon[53604]: from='osd.5 ' entity='osd.5' 2024-03-30T13:35:10.879 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:10 smithi159 ceph-mon[67002]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.6 MiB/s wr, 486 op/s; 170/1605 objects degraded (10.592%) 2024-03-30T13:35:10.879 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:10 smithi159 ceph-mon[67002]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:35:10.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:10 smithi159 ceph-mon[67002]: failed to encode map e128 with expected crc 2024-03-30T13:35:10.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:10 smithi159 ceph-mon[67002]: osd.5 [v2:172.21.15.187:6808/3777584294,v1:172.21.15.187:6809/3777584294] boot 2024-03-30T13:35:10.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:10 smithi159 ceph-mon[67002]: osdmap e128: 8 total, 8 up, 8 in 2024-03-30T13:35:10.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:10 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:35:10.880 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:10 smithi159 ceph-mon[67002]: failed to encode map e128 with expected crc 2024-03-30T13:35:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 285285 9.89 MB/sec execute 572 sec latency 846.651 ms 2024-03-30T13:35:11.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:10 smithi187 ceph-mon[53604]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.6 MiB/s wr, 486 op/s; 170/1605 objects degraded (10.592%) 2024-03-30T13:35:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:10 smithi187 ceph-mon[53604]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:35:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:10 smithi187 ceph-mon[53604]: failed to encode map e128 with expected crc 2024-03-30T13:35:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:10 smithi187 ceph-mon[53604]: osd.5 [v2:172.21.15.187:6808/3777584294,v1:172.21.15.187:6809/3777584294] boot 2024-03-30T13:35:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:10 smithi187 ceph-mon[53604]: osdmap e128: 8 total, 8 up, 8 in 2024-03-30T13:35:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:10 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-30T13:35:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:10 smithi187 ceph-mon[53604]: failed to encode map e128 with expected crc 2024-03-30T13:35:11.692 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:11 smithi159 ceph-mon[67002]: osdmap e129: 8 total, 8 up, 8 in 2024-03-30T13:35:11.692 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:11 smithi159 ceph-mon[67002]: failed to encode map e129 with expected crc 2024-03-30T13:35:11.692 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:11 smithi159 ceph-mon[67002]: failed to encode map e129 with expected crc 2024-03-30T13:35:11.692 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:11 smithi159 ceph-mon[67002]: failed to encode map e129 with expected crc 2024-03-30T13:35:11.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:11 smithi159 ceph-mon[67002]: failed to encode map e129 with expected crc 2024-03-30T13:35:11.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:11 smithi159 ceph-mon[67002]: failed to encode map e129 with expected crc 2024-03-30T13:35:11.693 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:11 smithi159 ceph-mon[67002]: failed to encode map e129 with expected crc 2024-03-30T13:35:11.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 285378 9.88 MB/sec execute 573 sec latency 1648.731 ms 2024-03-30T13:35:12.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:11 smithi187 ceph-mon[53604]: osdmap e129: 8 total, 8 up, 8 in 2024-03-30T13:35:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:11 smithi187 ceph-mon[53604]: failed to encode map e129 with expected crc 2024-03-30T13:35:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:11 smithi187 ceph-mon[53604]: failed to encode map e129 with expected crc 2024-03-30T13:35:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:11 smithi187 ceph-mon[53604]: failed to encode map e129 with expected crc 2024-03-30T13:35:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:11 smithi187 ceph-mon[53604]: failed to encode map e129 with expected crc 2024-03-30T13:35:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:11 smithi187 ceph-mon[53604]: failed to encode map e129 with expected crc 2024-03-30T13:35:12.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:11 smithi187 ceph-mon[53604]: failed to encode map e129 with expected crc 2024-03-30T13:35:12.721 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:12 smithi159 ceph-mon[67002]: pgmap v248: 97 pgs: 5 peering, 7 active+undersized, 19 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.5 MiB/s wr, 386 op/s; 143/1629 objects degraded (8.778%) 2024-03-30T13:35:12.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:12 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 143/1629 objects degraded (8.778%), 19 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:12.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:12 smithi159 ceph-mon[67002]: failed to encode map e130 with expected crc 2024-03-30T13:35:12.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:12 smithi159 ceph-mon[67002]: osdmap e130: 8 total, 8 up, 8 in 2024-03-30T13:35:12.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:12 smithi159 ceph-mon[67002]: failed to encode map e131 with expected crc 2024-03-30T13:35:12.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:12 smithi159 ceph-mon[67002]: osdmap e131: 8 total, 8 up, 8 in 2024-03-30T13:35:12.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:12.722 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:35:12.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 285635 9.87 MB/sec execute 574 sec latency 847.165 ms 2024-03-30T13:35:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:12 smithi187 ceph-mon[53604]: pgmap v248: 97 pgs: 5 peering, 7 active+undersized, 19 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.5 MiB/s wr, 386 op/s; 143/1629 objects degraded (8.778%) 2024-03-30T13:35:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:12 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 143/1629 objects degraded (8.778%), 19 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:12 smithi187 ceph-mon[53604]: failed to encode map e130 with expected crc 2024-03-30T13:35:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:12 smithi187 ceph-mon[53604]: osdmap e130: 8 total, 8 up, 8 in 2024-03-30T13:35:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:12 smithi187 ceph-mon[53604]: failed to encode map e131 with expected crc 2024-03-30T13:35:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:12 smithi187 ceph-mon[53604]: osdmap e131: 8 total, 8 up, 8 in 2024-03-30T13:35:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:35:13.737 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:13 smithi159 ceph-mon[67002]: failed to encode map e130 with expected crc 2024-03-30T13:35:13.738 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:13 smithi159 ceph-mon[67002]: failed to encode map e131 with expected crc 2024-03-30T13:35:13.738 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:13 smithi159 ceph-mon[67002]: failed to encode map e132 with expected crc 2024-03-30T13:35:13.738 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:13 smithi159 ceph-mon[67002]: osdmap e132: 8 total, 8 up, 8 in 2024-03-30T13:35:13.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 285950 9.88 MB/sec execute 575 sec latency 921.828 ms 2024-03-30T13:35:14.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:13 smithi187 ceph-mon[53604]: failed to encode map e130 with expected crc 2024-03-30T13:35:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:13 smithi187 ceph-mon[53604]: failed to encode map e131 with expected crc 2024-03-30T13:35:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:13 smithi187 ceph-mon[53604]: failed to encode map e132 with expected crc 2024-03-30T13:35:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:13 smithi187 ceph-mon[53604]: osdmap e132: 8 total, 8 up, 8 in 2024-03-30T13:35:14.773 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:14 smithi159 ceph-mon[67002]: failed to encode map e132 with expected crc 2024-03-30T13:35:14.774 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:14 smithi159 ceph-mon[67002]: pgmap v252: 97 pgs: 5 peering, 4 active+undersized, 1 active+undersized+remapped, 17 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s wr, 84 op/s; 119/1641 objects degraded (7.252%); 231 KiB/s, 11 objects/s recovering 2024-03-30T13:35:14.774 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:14 smithi159 ceph-mon[67002]: osdmap e133: 8 total, 8 up, 8 in 2024-03-30T13:35:14.774 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:14 smithi159 ceph-mon[67002]: failed to encode map e133 with expected crc 2024-03-30T13:35:14.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 286275 9.87 MB/sec execute 576 sec latency 484.521 ms 2024-03-30T13:35:15.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:14 smithi187 ceph-mon[53604]: failed to encode map e132 with expected crc 2024-03-30T13:35:15.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:14 smithi187 ceph-mon[53604]: pgmap v252: 97 pgs: 5 peering, 4 active+undersized, 1 active+undersized+remapped, 17 active+undersized+degraded, 70 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s wr, 84 op/s; 119/1641 objects degraded (7.252%); 231 KiB/s, 11 objects/s recovering 2024-03-30T13:35:15.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:14 smithi187 ceph-mon[53604]: osdmap e133: 8 total, 8 up, 8 in 2024-03-30T13:35:15.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:14 smithi187 ceph-mon[53604]: failed to encode map e133 with expected crc 2024-03-30T13:35:15.779 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:15 smithi159 ceph-mon[67002]: pgmap v254: 97 pgs: 5 peering, 1 active+undersized+remapped, 91 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 467 op/s; 1.7 MiB/s, 42 objects/s recovering 2024-03-30T13:35:15.779 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:15 smithi159 ceph-mon[67002]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 119/1641 objects degraded (7.252%), 17 pgs degraded) 2024-03-30T13:35:15.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 286581 9.87 MB/sec execute 577 sec latency 42.210 ms 2024-03-30T13:35:16.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:15 smithi187 ceph-mon[53604]: pgmap v254: 97 pgs: 5 peering, 1 active+undersized+remapped, 91 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 467 op/s; 1.7 MiB/s, 42 objects/s recovering 2024-03-30T13:35:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:15 smithi187 ceph-mon[53604]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 119/1641 objects degraded (7.252%), 17 pgs degraded) 2024-03-30T13:35:16.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 286921 9.87 MB/sec execute 578 sec latency 52.347 ms 2024-03-30T13:35:17.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 287201 9.86 MB/sec execute 579 sec latency 52.052 ms 2024-03-30T13:35:18.327 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:18 smithi159 ceph-mon[67002]: pgmap v255: 97 pgs: 2 peering, 1 active+undersized+remapped, 94 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.3 MiB/s wr, 356 op/s; 1.3 MiB/s, 34 objects/s recovering 2024-03-30T13:35:18.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:18 smithi187 ceph-mon[53604]: pgmap v255: 97 pgs: 2 peering, 1 active+undersized+remapped, 94 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.3 MiB/s wr, 356 op/s; 1.3 MiB/s, 34 objects/s recovering 2024-03-30T13:35:18.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 287443 9.86 MB/sec execute 580 sec latency 97.680 ms 2024-03-30T13:35:19.417 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:35:19.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 287712 9.86 MB/sec execute 581 sec latency 59.263 ms 2024-03-30T13:35:20.273 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (8m) 93s ago 18m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (4m) 93s ago 18m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (4m) 18s ago 17m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 93s ago 17m - - 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (12m) 93s ago 12m 38.5M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (12m) 18s ago 12m 17.7M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (7m) 93s ago 19m 507M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (6m) 18s ago 17m 451M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (6m) 93s ago 19m 49.7M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (5m) 18s ago 17m 42.8M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (11m) 93s ago 11m 283M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:35:20.274 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (11m) 18s ago 11m 86.5M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (8m) 93s ago 17m 13.0M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (7m) 18s ago 17m 20.2M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (4m) 93s ago 16m 356M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (3m) 93s ago 16m 277M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (2m) 93s ago 15m 241M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (101s) 93s ago 15m 12.0M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (59s) 18s ago 14m 285M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (22s) 18s ago 14m 12.6M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (14m) 18s ago 14m 626M 2704M 17.2.0 e1d6a67b021e 9ab871f1994c 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (13m) 18s ago 13m 637M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:35:20.275 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (6m) 93s ago 16m 63.2M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:35:20.430 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:20 smithi159 ceph-mon[67002]: pgmap v256: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.1 MiB/s wr, 383 op/s; 1.1 MiB/s, 30 objects/s recovering 2024-03-30T13:35:20.431 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:20 smithi159 ceph-mon[67002]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:20.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:20 smithi187 ceph-mon[53604]: pgmap v256: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.1 MiB/s wr, 383 op/s; 1.1 MiB/s, 30 objects/s recovering 2024-03-30T13:35:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:20 smithi187 ceph-mon[53604]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:20.784 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 6 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:35:20.785 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:35:20.786 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:35:20.786 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:35:20.786 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-30T13:35:20.786 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 10 2024-03-30T13:35:20.786 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:35:20.786 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:35:20.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 287882 9.85 MB/sec execute 582 sec latency 99.225 ms 2024-03-30T13:35:21.219 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:35:21.219 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:35:21.219 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:35:21.219 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:35:21.219 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:35:21.219 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:35:21.219 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:35:21.219 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:35:21.220 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:35:21.220 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "12/23 daemons upgraded", 2024-03-30T13:35:21.220 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:35:21.220 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:35:21.220 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:35:21.453 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:21 smithi159 ceph-mon[67002]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:21.453 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:21 smithi159 ceph-mon[67002]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:21.453 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:21 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1750990456' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:21.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:21 smithi187 ceph-mon[53604]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:21.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:21 smithi187 ceph-mon[53604]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:21.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:21 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1750990456' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:21.898 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:35:21.898 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:35:21.898 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:35:21.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 288151 9.85 MB/sec execute 583 sec latency 52.867 ms 2024-03-30T13:35:22.223 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:22 smithi159 ceph-mon[67002]: pgmap v257: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 456 op/s; 910 KiB/s, 21 objects/s recovering 2024-03-30T13:35:22.223 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:22 smithi159 ceph-mon[67002]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:22.223 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:22 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/840404203' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:35:22.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:22 smithi187 ceph-mon[53604]: pgmap v257: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 456 op/s; 910 KiB/s, 21 objects/s recovering 2024-03-30T13:35:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:22 smithi187 ceph-mon[53604]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:22.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:22 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/840404203' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:35:22.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 288482 9.84 MB/sec execute 584 sec latency 38.713 ms 2024-03-30T13:35:23.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-30T13:35:23.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:23 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-30T13:35:23.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:23 smithi159 ceph-mon[67002]: Upgrade: osd.6 is safe to restart 2024-03-30T13:35:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-30T13:35:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:23 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-30T13:35:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:23 smithi187 ceph-mon[53604]: Upgrade: osd.6 is safe to restart 2024-03-30T13:35:23.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 288791 9.84 MB/sec execute 585 sec latency 41.630 ms 2024-03-30T13:35:24.254 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:24 smithi187 ceph-mon[53604]: pgmap v258: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.0 MiB/s wr, 389 op/s; 741 KiB/s, 17 objects/s recovering 2024-03-30T13:35:24.273 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:24 smithi159 ceph-mon[67002]: pgmap v258: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.0 MiB/s wr, 389 op/s; 741 KiB/s, 17 objects/s recovering 2024-03-30T13:35:24.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 289094 9.83 MB/sec execute 586 sec latency 45.805 ms 2024-03-30T13:35:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:25 smithi187 ceph-mon[53604]: Upgrade: Updating osd.6 2024-03-30T13:35:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-30T13:35:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:35:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:25 smithi187 ceph-mon[53604]: Deploying daemon osd.6 on smithi187 2024-03-30T13:35:25.548 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:25 smithi159 ceph-mon[67002]: Upgrade: Updating osd.6 2024-03-30T13:35:25.548 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:25.548 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-30T13:35:25.548 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:35:25.548 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:25 smithi159 ceph-mon[67002]: Deploying daemon osd.6 on smithi187 2024-03-30T13:35:25.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 289651 9.84 MB/sec execute 587 sec latency 44.471 ms 2024-03-30T13:35:26.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:26 smithi187 ceph-mon[53604]: pgmap v259: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 417 op/s; 179 KiB/s, 6 objects/s recovering 2024-03-30T13:35:26.566 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:26 smithi159 ceph-mon[67002]: pgmap v259: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 417 op/s; 179 KiB/s, 6 objects/s recovering 2024-03-30T13:35:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 290122 9.85 MB/sec execute 588 sec latency 35.843 ms 2024-03-30T13:35:27.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 290484 9.85 MB/sec execute 589 sec latency 49.436 ms 2024-03-30T13:35:28.499 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:28 smithi187 systemd[1]: Stopping Ceph osd.6 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:35:28.622 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:28 smithi159 ceph-mon[67002]: pgmap v260: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.9 MiB/s wr, 374 op/s; 93 KiB/s, 3 objects/s recovering 2024-03-30T13:35:28.622 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:28.622 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:35:28.622 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:28 smithi187 ceph-mon[53604]: pgmap v260: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.9 MiB/s wr, 374 op/s; 93 KiB/s, 3 objects/s recovering 2024-03-30T13:35:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:35:28.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:28.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[37719]: 2024-03-30T13:35:28.556+0000 7f8490a29700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:35:28.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[37719]: 2024-03-30T13:35:28.556+0000 7f8490a29700 -1 osd.6 133 *** Got signal Terminated *** 2024-03-30T13:35:28.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[37719]: 2024-03-30T13:35:28.556+0000 7f8490a29700 -1 osd.6 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:35:28.941 INFO:teuthology.orchestra.run.smithi159.stdout: 5 290749 9.84 MB/sec execute 590 sec latency 75.668 ms 2024-03-30T13:35:29.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:29 smithi159 ceph-mon[67002]: osd.6 marked itself down and dead 2024-03-30T13:35:29.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:29 smithi187 ceph-mon[53604]: osd.6 marked itself down and dead 2024-03-30T13:35:29.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 290853 9.83 MB/sec execute 591 sec latency 448.547 ms 2024-03-30T13:35:30.454 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:30 smithi187 podman[62574]: 2024-03-30 13:35:30.175701728 +0000 UTC m=+1.758255165 container died 9ab871f1994cc383866021771978fdecc8ee8871c61474c203a66284e85b2318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, io.openshift.expose-services=, com.redhat.component=centos-stream-container, architecture=x86_64, vendor=Red Hat, Inc., ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, distribution-scope=public, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0) 2024-03-30T13:35:30.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:30 smithi159 ceph-mon[67002]: pgmap v261: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 526 op/s; 65 KiB/s, 2 objects/s recovering 2024-03-30T13:35:30.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:30 smithi159 ceph-mon[67002]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:35:30.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:30 smithi159 ceph-mon[67002]: osdmap e134: 8 total, 7 up, 8 in 2024-03-30T13:35:30.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:30 smithi159 ceph-mon[67002]: failed to encode map e134 with expected crc 2024-03-30T13:35:30.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:30 smithi159 ceph-mon[67002]: failed to encode map e134 with expected crc 2024-03-30T13:35:30.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:30 smithi159 ceph-mon[67002]: failed to encode map e134 with expected crc 2024-03-30T13:35:30.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:30 smithi187 ceph-mon[53604]: pgmap v261: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 526 op/s; 65 KiB/s, 2 objects/s recovering 2024-03-30T13:35:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:30 smithi187 ceph-mon[53604]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:35:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:30 smithi187 ceph-mon[53604]: osdmap e134: 8 total, 7 up, 8 in 2024-03-30T13:35:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:30 smithi187 ceph-mon[53604]: failed to encode map e134 with expected crc 2024-03-30T13:35:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:30 smithi187 ceph-mon[53604]: failed to encode map e134 with expected crc 2024-03-30T13:35:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:30 smithi187 ceph-mon[53604]: failed to encode map e134 with expected crc 2024-03-30T13:35:30.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:30 smithi187 podman[62574]: 2024-03-30 13:35:30.553110061 +0000 UTC m=+2.135663494 container cleanup 9ab871f1994cc383866021771978fdecc8ee8871c61474c203a66284e85b2318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vcs-type=git) 2024-03-30T13:35:30.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:30 smithi187 bash[62574]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6 2024-03-30T13:35:30.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 291010 9.82 MB/sec execute 592 sec latency 1075.451 ms 2024-03-30T13:35:31.519 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:31 smithi159 ceph-mon[67002]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.519 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:31 smithi159 ceph-mon[67002]: osdmap e135: 8 total, 7 up, 8 in 2024-03-30T13:35:31.519 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:31 smithi159 ceph-mon[67002]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.520 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:31 smithi159 ceph-mon[67002]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.520 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:31 smithi159 ceph-mon[67002]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.520 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:31 smithi159 ceph-mon[67002]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.708 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:31 smithi187 podman[62626]: 2024-03-30 13:35:31.296610256 +0000 UTC m=+1.117218563 container remove 9ab871f1994cc383866021771978fdecc8ee8871c61474c203a66284e85b2318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, version=8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, ceph=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, name=centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-03-30T13:35:31.708 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:31 smithi187 ceph-mon[53604]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.708 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:31 smithi187 ceph-mon[53604]: osdmap e135: 8 total, 7 up, 8 in 2024-03-30T13:35:31.708 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:31 smithi187 ceph-mon[53604]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.708 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:31 smithi187 ceph-mon[53604]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.708 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:31 smithi187 ceph-mon[53604]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.708 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:31 smithi187 ceph-mon[53604]: failed to encode map e135 with expected crc 2024-03-30T13:35:31.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 291391 9.83 MB/sec execute 593 sec latency 33.397 ms 2024-03-30T13:35:32.001 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:31 smithi187 podman[62711]: 2024-03-30 13:35:31.611380101 +0000 UTC m=+0.021773405 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:35:32.002 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:31 smithi187 podman[62711]: 2024-03-30 13:35:31.747838113 +0000 UTC m=+0.158231409 container create 5493bd945a5cc2009c58a1262acb56f207cd1dc5ea0615ce7890248cdeadea0f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-30T13:35:32.002 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:32 smithi187 podman[62711]: 2024-03-30 13:35:32.000719685 +0000 UTC m=+0.411112981 container init 5493bd945a5cc2009c58a1262acb56f207cd1dc5ea0615ce7890248cdeadea0f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:35:32.272 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:32 smithi187 podman[62711]: 2024-03-30 13:35:32.005191267 +0000 UTC m=+0.415584572 container start 5493bd945a5cc2009c58a1262acb56f207cd1dc5ea0615ce7890248cdeadea0f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-30T13:35:32.272 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:32 smithi187 podman[62711]: 2024-03-30 13:35:32.100837577 +0000 UTC m=+0.511230876 container attach 5493bd945a5cc2009c58a1262acb56f207cd1dc5ea0615ce7890248cdeadea0f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-30T13:35:32.529 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:32 smithi159 ceph-mon[67002]: pgmap v264: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 7 peering, 8 stale+active+clean, 78 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.4 MiB/s wr, 586 op/s; 22/1695 objects degraded (1.298%) 2024-03-30T13:35:32.529 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:32 smithi159 ceph-mon[67002]: Health check failed: Degraded data redundancy: 22/1695 objects degraded (1.298%), 3 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:32.751 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:32 smithi187 ceph-mon[53604]: pgmap v264: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 7 peering, 8 stale+active+clean, 78 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.4 MiB/s wr, 586 op/s; 22/1695 objects degraded (1.298%) 2024-03-30T13:35:32.751 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:32 smithi187 ceph-mon[53604]: Health check failed: Degraded data redundancy: 22/1695 objects degraded (1.298%), 3 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:32.752 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:32 smithi187 podman[62711]: 2024-03-30 13:35:32.35613888 +0000 UTC m=+0.766532186 container died 5493bd945a5cc2009c58a1262acb56f207cd1dc5ea0615ce7890248cdeadea0f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-03-30T13:35:32.940 INFO:teuthology.orchestra.run.smithi159.stdout: 5 291835 9.83 MB/sec execute 594 sec latency 34.938 ms 2024-03-30T13:35:33.494 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 podman[62737]: 2024-03-30 13:35:33.239048946 +0000 UTC m=+0.877500896 container remove 5493bd945a5cc2009c58a1262acb56f207cd1dc5ea0615ce7890248cdeadea0f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-30T13:35:33.494 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.6.service: Deactivated successfully. 2024-03-30T13:35:33.494 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 systemd[1]: Stopped Ceph osd.6 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:35:33.494 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.6.service: Consumed 5min 45.962s CPU time. 2024-03-30T13:35:33.750 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 systemd[1]: Starting Ceph osd.6 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:35:33.750 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 podman[62842]: 2024-03-30 13:35:33.627182942 +0000 UTC m=+0.020407677 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:35:33.750 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 podman[62842]: 2024-03-30 13:35:33.749087376 +0000 UTC m=+0.142312101 container create bf68398dbff97b21a073678f1977d0ab15dd51e84a90ea0f141bed863d95c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5) 2024-03-30T13:35:33.941 INFO:teuthology.orchestra.run.smithi159.stdout: 5 292189 9.82 MB/sec execute 595 sec latency 38.030 ms 2024-03-30T13:35:34.022 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 podman[62842]: 2024-03-30 13:35:33.985218825 +0000 UTC m=+0.378443557 container init bf68398dbff97b21a073678f1977d0ab15dd51e84a90ea0f141bed863d95c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc) 2024-03-30T13:35:34.022 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:33 smithi187 podman[62842]: 2024-03-30 13:35:33.989365979 +0000 UTC m=+0.382590707 container start bf68398dbff97b21a073678f1977d0ab15dd51e84a90ea0f141bed863d95c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-03-30T13:35:34.505 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 podman[62842]: 2024-03-30 13:35:34.085501706 +0000 UTC m=+0.478726445 container attach bf68398dbff97b21a073678f1977d0ab15dd51e84a90ea0f141bed863d95c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-30T13:35:34.674 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:34 smithi159 ceph-mon[67002]: pgmap v265: 97 pgs: 6 active+undersized, 10 active+undersized+degraded, 7 peering, 3 stale+active+clean, 71 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.6 MiB/s wr, 431 op/s; 78/1611 objects degraded (4.842%); 500 B/s, 0 objects/s recovering 2024-03-30T13:35:34.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:34 smithi187 ceph-mon[53604]: pgmap v265: 97 pgs: 6 active+undersized, 10 active+undersized+degraded, 7 peering, 3 stale+active+clean, 71 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.6 MiB/s wr, 431 op/s; 78/1611 objects degraded (4.842%); 500 B/s, 0 objects/s recovering 2024-03-30T13:35:34.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate[62857]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-30T13:35:34.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 bash[62842]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-30T13:35:34.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 bash[62842]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-30T13:35:34.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 bash[62842]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-30T13:35:34.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 bash[62842]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-30T13:35:34.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 bash[62842]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 bash[62842]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 bash[62842]: --> ceph-volume raw activate successful for osd ID: 6 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate[62857]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate[62857]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate[62857]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate[62857]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate[62857]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate[62857]: --> ceph-volume raw activate successful for osd ID: 6 2024-03-30T13:35:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:34 smithi187 podman[62842]: 2024-03-30 13:35:34.532259695 +0000 UTC m=+0.925484410 container died bf68398dbff97b21a073678f1977d0ab15dd51e84a90ea0f141bed863d95c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-03-30T13:35:34.941 INFO:teuthology.orchestra.run.smithi159.stdout: 5 292519 9.82 MB/sec execute 596 sec latency 44.972 ms 2024-03-30T13:35:35.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:35 smithi187 podman[62989]: 2024-03-30 13:35:35.407973294 +0000 UTC m=+0.870571483 container remove bf68398dbff97b21a073678f1977d0ab15dd51e84a90ea0f141bed863d95c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-30T13:35:35.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:35 smithi187 podman[63052]: 2024-03-30 13:35:35.538984038 +0000 UTC m=+0.021475525 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:35:35.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:35 smithi187 podman[63052]: 2024-03-30 13:35:35.650654785 +0000 UTC m=+0.133146257 container create e4f91e53cf7d2dc3008993b75948c20ca8882c7a87d8df889c7fafbadc9b83ec (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, ceph=True) 2024-03-30T13:35:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: 5 292989 9.83 MB/sec execute 597 sec latency 72.404 ms 2024-03-30T13:35:36.522 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:36 smithi187 podman[63052]: 2024-03-30 13:35:36.123814956 +0000 UTC m=+0.606306443 container init e4f91e53cf7d2dc3008993b75948c20ca8882c7a87d8df889c7fafbadc9b83ec (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:35:36.522 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:36 smithi187 podman[63052]: 2024-03-30 13:35:36.128476482 +0000 UTC m=+0.610967962 container start e4f91e53cf7d2dc3008993b75948c20ca8882c7a87d8df889c7fafbadc9b83ec (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326) 2024-03-30T13:35:36.522 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:36 smithi187 ceph-osd[63070]: -- 172.21.15.187:0/101635229 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564f2d439680 con 0x564f2d402c00 2024-03-30T13:35:36.522 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:36 smithi187 bash[63052]: e4f91e53cf7d2dc3008993b75948c20ca8882c7a87d8df889c7fafbadc9b83ec 2024-03-30T13:35:36.523 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:36 smithi187 systemd[1]: Started Ceph osd.6 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:35:36.715 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:36 smithi159 ceph-mon[67002]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 567 op/s; 200/1650 objects degraded (12.121%); 500 B/s, 0 objects/s recovering 2024-03-30T13:35:36.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:36.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:36.716 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:36 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:35:36.941 INFO:teuthology.orchestra.run.smithi159.stdout: 5 293408 9.83 MB/sec execute 598 sec latency 45.336 ms 2024-03-30T13:35:37.022 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:35:36.681+0000 7f73361277c0 -1 Falling back to public interface 2024-03-30T13:35:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:36 smithi187 ceph-mon[53604]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 567 op/s; 200/1650 objects degraded (12.121%); 500 B/s, 0 objects/s recovering 2024-03-30T13:35:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:36 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:35:37.734 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:37 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 200/1650 objects degraded (12.121%), 25 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:37.941 INFO:teuthology.orchestra.run.smithi159.stdout: 5 293757 9.83 MB/sec execute 599 sec latency 38.878 ms 2024-03-30T13:35:38.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:37 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 200/1650 objects degraded (12.121%), 25 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:38.746 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:38 smithi159 ceph-mon[67002]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.4 MiB/s wr, 461 op/s; 197/1662 objects degraded (11.853%); 500 B/s, 0 objects/s recovering 2024-03-30T13:35:38.943 INFO:teuthology.orchestra.run.smithi159.stdout: 5 cleanup 600 sec 2024-03-30T13:35:39.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:38 smithi187 ceph-mon[53604]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.4 MiB/s wr, 461 op/s; 197/1662 objects degraded (11.853%); 500 B/s, 0 objects/s recovering 2024-03-30T13:35:39.077 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs00000100000002d90000000a' failed - Device or resource busy 2024-03-30T13:35:39.077 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000032300000013' failed - Device or resource busy 2024-03-30T13:35:39.078 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000cb7000000002' failed - Device or resource busy 2024-03-30T13:35:39.078 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000030f00000011' failed - Device or resource busy 2024-03-30T13:35:39.080 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs00000100000002f100000010' failed - Device or resource busy 2024-03-30T13:35:39.080 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs00000100000002be00000015' failed - Device or resource busy 2024-03-30T13:35:39.080 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000031a0000000c' failed - Device or resource busy 2024-03-30T13:35:39.083 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000031e0000000e' failed - Device or resource busy 2024-03-30T13:35:39.090 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000030900000017' failed - Device or resource busy 2024-03-30T13:35:39.147 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs00000100000002c500000001' failed - Device or resource busy 2024-03-30T13:35:39.149 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs00000100000002ca00000006' failed - Device or resource busy 2024-03-30T13:35:39.150 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs00000100000002e700000007' failed - Device or resource busy 2024-03-30T13:35:39.162 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs00000100000002cf00000016' failed - Device or resource busy 2024-03-30T13:35:39.175 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000cb4e00000004' failed - Device or resource busy 2024-03-30T13:35:39.178 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs00000100000002f600000014' failed - Device or resource busy 2024-03-30T13:35:39.180 INFO:teuthology.orchestra.run.smithi159.stdout:[296529] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000030500000008' failed - Device or resource busy 2024-03-30T13:35:39.258 INFO:teuthology.orchestra.run.smithi159.stdout: 0 cleanup 600 sec 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Operation Count AvgLat MaxLat 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: ---------------------------------------- 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: NTCreateX 189254 4.547 1075.444 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Close 139039 0.138 9.662 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Rename 8027 9.456 101.228 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Unlink 38200 6.992 1051.676 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Qpathinfo 171807 1.160 136.989 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Qfileinfo 29970 0.008 1.466 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Qfsinfo 31431 0.614 25.571 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Sfileinfo 15461 12.771 1049.063 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: Find 66329 0.977 15.603 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: WriteX 93571 13.778 2070.472 2024-03-30T13:35:39.259 INFO:teuthology.orchestra.run.smithi159.stdout: ReadX 296900 0.010 6.714 2024-03-30T13:35:39.260 INFO:teuthology.orchestra.run.smithi159.stdout: LockX 618 0.377 3.578 2024-03-30T13:35:39.260 INFO:teuthology.orchestra.run.smithi159.stdout: UnlockX 618 0.340 4.313 2024-03-30T13:35:39.260 INFO:teuthology.orchestra.run.smithi159.stdout: Flush 13272 0.002 0.047 2024-03-30T13:35:39.260 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:35:39.260 INFO:teuthology.orchestra.run.smithi159.stdout:Throughput 9.82957 MB/sec 5 clients 5 procs max_latency=2070.476 ms 2024-03-30T13:35:39.262 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-03-30T13:35:39.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-03-30T13:35:39.464 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-03-30T13:35:39.510 INFO:teuthology.orchestra.run.smithi159.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-30T13:35:39.510 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:35:39.510 INFO:teuthology.orchestra.run.smithi159.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-03-30T13:35:39.516 INFO:teuthology.orchestra.run.smithi159.stdout:3 of 5 processes prepared for launch 0 sec 2024-03-30T13:35:39.611 INFO:teuthology.orchestra.run.smithi159.stdout:5 of 5 processes prepared for launch 0 sec 2024-03-30T13:35:39.611 INFO:teuthology.orchestra.run.smithi159.stdout:releasing clients 2024-03-30T13:35:40.803 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:40 smithi159 ceph-mon[67002]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 455 op/s; 193/1677 objects degraded (11.509%); 415 B/s, 0 objects/s recovering 2024-03-30T13:35:41.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:40 smithi187 ceph-mon[53604]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 455 op/s; 193/1677 objects degraded (11.509%); 415 B/s, 0 objects/s recovering 2024-03-30T13:35:41.586 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:41 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:41.587 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:41 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:41.611 INFO:teuthology.orchestra.run.smithi159.stdout: 5 339 19.77 MB/sec execute 1 sec latency 77.627 ms 2024-03-30T13:35:42.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:41 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:42.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:41 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:42.611 INFO:teuthology.orchestra.run.smithi159.stdout: 5 467 20.29 MB/sec execute 2 sec latency 47.968 ms 2024-03-30T13:35:42.614 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:42 smithi159 ceph-mon[67002]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 418 op/s; 192/1674 objects degraded (11.470%); 376 B/s, 0 objects/s recovering 2024-03-30T13:35:42.614 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:42.614 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:42.614 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:42 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 192/1674 objects degraded (11.470%), 25 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:42.614 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:42.614 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:35:42.849 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:42 smithi187 ceph-mon[53604]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 418 op/s; 192/1674 objects degraded (11.470%); 376 B/s, 0 objects/s recovering 2024-03-30T13:35:42.850 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:42.850 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:42.850 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:42 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 192/1674 objects degraded (11.470%), 25 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:42.850 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:42.850 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:35:43.611 INFO:teuthology.orchestra.run.smithi159.stdout: 5 717 21.64 MB/sec execute 3 sec latency 75.659 ms 2024-03-30T13:35:44.522 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:35:44.412+0000 7f73361277c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-03-30T13:35:44.611 INFO:teuthology.orchestra.run.smithi159.stdout: 5 1038 18.04 MB/sec execute 4 sec latency 30.445 ms 2024-03-30T13:35:44.672 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:44 smithi159 ceph-mon[67002]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 6.5 MiB/s wr, 440 op/s; 177/1518 objects degraded (11.660%); 333 B/s, 0 objects/s recovering 2024-03-30T13:35:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:44 smithi187 ceph-mon[53604]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 6.5 MiB/s wr, 440 op/s; 177/1518 objects degraded (11.660%); 333 B/s, 0 objects/s recovering 2024-03-30T13:35:45.611 INFO:teuthology.orchestra.run.smithi159.stdout: 5 cleanup 5 sec 2024-03-30T13:35:45.799 INFO:teuthology.orchestra.run.smithi159.stdout: 0 cleanup 5 sec 2024-03-30T13:35:45.799 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:35:45.799 INFO:teuthology.orchestra.run.smithi159.stdout: Operation Count AvgLat MaxLat 2024-03-30T13:35:45.799 INFO:teuthology.orchestra.run.smithi159.stdout: ---------------------------------------- 2024-03-30T13:35:45.799 INFO:teuthology.orchestra.run.smithi159.stdout: NTCreateX 811 8.317 74.469 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Close 751 8.221 75.651 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Rename 50 10.929 30.435 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Unlink 85 3.809 20.197 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Qpathinfo 483 1.549 27.211 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Qfileinfo 354 17.675 46.886 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Qfsinfo 162 0.447 2.183 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Sfileinfo 170 14.125 30.391 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Find 228 0.778 3.476 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: WriteX 1657 0.026 0.066 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: ReadX 1026 0.019 0.510 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: Flush 120 12.785 27.158 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:35:45.800 INFO:teuthology.orchestra.run.smithi159.stdout:Throughput 18.0435 MB/sec 5 clients 5 procs max_latency=77.627 ms 2024-03-30T13:35:45.801 DEBUG:teuthology.parallel:result is None 2024-03-30T13:35:46.271 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:45 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:35:45.918+0000 7f73361277c0 -1 osd.6 133 log_to_monitors true 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 581 op/s; 166/1350 objects degraded (12.296%); 0 B/s, 0 objects/s recovering 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='osd.6 [v2:172.21.15.187:6816/331706955,v1:172.21.15.187:6817/331706955]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:35:46.883 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:46.884 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:46.884 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:46.884 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 581 op/s; 166/1350 objects degraded (12.296%); 0 B/s, 0 objects/s recovering 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='osd.6 [v2:172.21.15.187:6816/331706955,v1:172.21.15.187:6817/331706955]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:35:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:47.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:47.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:47.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-30T13:35:47.272 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:35:46 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:35:46.881+0000 7f732757f700 -1 osd.6 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 12 MiB/s wr, 537 op/s; 153/1290 objects degraded (11.860%) 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 14 MiB/s wr, 531 op/s; 153/1290 objects degraded (11.860%) 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: osdmap e136: 8 total, 7 up, 8 in 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: from='osd.6 [v2:172.21.15.187:6816/331706955,v1:172.21.15.187:6817/331706955]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:35:48.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:47 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 153/1290 objects degraded (11.860%), 25 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 12 MiB/s wr, 537 op/s; 153/1290 objects degraded (11.860%) 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 14 MiB/s wr, 531 op/s; 153/1290 objects degraded (11.860%) 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: osdmap e136: 8 total, 7 up, 8 in 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: from='osd.6 [v2:172.21.15.187:6816/331706955,v1:172.21.15.187:6817/331706955]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:35:48.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:47 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 153/1290 objects degraded (11.860%), 25 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:49.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:48 smithi159 ceph-mon[67002]: failed to encode map e136 with expected crc 2024-03-30T13:35:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:48 smithi159 ceph-mon[67002]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:35:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:48 smithi159 ceph-mon[67002]: osd.6 [v2:172.21.15.187:6816/331706955,v1:172.21.15.187:6817/331706955] boot 2024-03-30T13:35:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:48 smithi159 ceph-mon[67002]: osdmap e137: 8 total, 8 up, 8 in 2024-03-30T13:35:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:48 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:35:49.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:48 smithi187 ceph-mon[53604]: failed to encode map e136 with expected crc 2024-03-30T13:35:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:48 smithi187 ceph-mon[53604]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:35:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:48 smithi187 ceph-mon[53604]: osd.6 [v2:172.21.15.187:6816/331706955,v1:172.21.15.187:6817/331706955] boot 2024-03-30T13:35:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:48 smithi187 ceph-mon[53604]: osdmap e137: 8 total, 8 up, 8 in 2024-03-30T13:35:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:48 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-30T13:35:50.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:49 smithi159 ceph-mon[67002]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 18 MiB/s wr, 601 op/s; 163/1395 objects degraded (11.685%) 2024-03-30T13:35:50.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:49 smithi159 ceph-mon[67002]: failed to encode map e137 with expected crc 2024-03-30T13:35:50.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:49 smithi159 ceph-mon[67002]: osdmap e138: 8 total, 8 up, 8 in 2024-03-30T13:35:50.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:49 smithi187 ceph-mon[53604]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 18 MiB/s wr, 601 op/s; 163/1395 objects degraded (11.685%) 2024-03-30T13:35:50.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:49 smithi187 ceph-mon[53604]: failed to encode map e137 with expected crc 2024-03-30T13:35:50.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:49 smithi187 ceph-mon[53604]: osdmap e138: 8 total, 8 up, 8 in 2024-03-30T13:35:51.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:50 smithi159 ceph-mon[67002]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:50 smithi159 ceph-mon[67002]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:50 smithi159 ceph-mon[67002]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:50 smithi159 ceph-mon[67002]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:50 smithi159 ceph-mon[67002]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:50 smithi159 ceph-mon[67002]: osdmap e139: 8 total, 8 up, 8 in 2024-03-30T13:35:51.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:50 smithi187 ceph-mon[53604]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:50 smithi187 ceph-mon[53604]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:50 smithi187 ceph-mon[53604]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:50 smithi187 ceph-mon[53604]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:50 smithi187 ceph-mon[53604]: failed to encode map e138 with expected crc 2024-03-30T13:35:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:50 smithi187 ceph-mon[53604]: osdmap e139: 8 total, 8 up, 8 in 2024-03-30T13:35:52.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:51 smithi159 ceph-mon[67002]: pgmap v278: 97 pgs: 4 remapped+peering, 9 peering, 7 active+undersized, 12 active+undersized+degraded, 65 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s wr, 172 op/s; 70/1413 objects degraded (4.954%) 2024-03-30T13:35:52.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:51 smithi159 ceph-mon[67002]: failed to encode map e139 with expected crc 2024-03-30T13:35:52.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:51 smithi159 ceph-mon[67002]: failed to encode map e139 with expected crc 2024-03-30T13:35:52.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:51 smithi159 ceph-mon[67002]: osdmap e140: 8 total, 8 up, 8 in 2024-03-30T13:35:52.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:51 smithi187 ceph-mon[53604]: pgmap v278: 97 pgs: 4 remapped+peering, 9 peering, 7 active+undersized, 12 active+undersized+degraded, 65 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s wr, 172 op/s; 70/1413 objects degraded (4.954%) 2024-03-30T13:35:52.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:51 smithi187 ceph-mon[53604]: failed to encode map e139 with expected crc 2024-03-30T13:35:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:51 smithi187 ceph-mon[53604]: failed to encode map e139 with expected crc 2024-03-30T13:35:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:51 smithi187 ceph-mon[53604]: osdmap e140: 8 total, 8 up, 8 in 2024-03-30T13:35:52.273 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (9m) 2m ago 18m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (5m) 2m ago 18m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (5m) 12s ago 17m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 2m ago 18m - - 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (12m) 2m ago 12m 38.5M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (12m) 12s ago 12m 17.8M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (8m) 2m ago 19m 507M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (7m) 12s ago 17m 452M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:35:52.955 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (6m) 2m ago 19m 49.7M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (6m) 12s ago 17m 43.9M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (12m) 2m ago 12m 283M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (12m) 12s ago 12m 86.5M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (8m) 2m ago 18m 13.0M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (8m) 12s ago 17m 17.7M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (4m) 2m ago 17m 356M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (3m) 2m ago 16m 277M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (3m) 2m ago 16m 241M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (2m) 2m ago 15m 12.0M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (92s) 12s ago 15m 326M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (55s) 12s ago 14m 292M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (17s) 12s ago 14m 15.1M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e4f91e53cf7d 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (14m) 12s ago 14m 658M 2704M 17.2.0 e1d6a67b021e 51b657e42788 2024-03-30T13:35:52.956 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (7m) 2m ago 17m 63.2M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:35:52.957 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:52 smithi159 ceph-mon[67002]: failed to encode map e140 with expected crc 2024-03-30T13:35:52.957 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:52 smithi159 ceph-mon[67002]: osdmap e141: 8 total, 8 up, 8 in 2024-03-30T13:35:52.957 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:52 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 70/1413 objects degraded (4.954%), 12 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:52.957 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:52 smithi159 ceph-mon[67002]: osdmap e142: 8 total, 8 up, 8 in 2024-03-30T13:35:53.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:52 smithi187 ceph-mon[53604]: failed to encode map e140 with expected crc 2024-03-30T13:35:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:52 smithi187 ceph-mon[53604]: osdmap e141: 8 total, 8 up, 8 in 2024-03-30T13:35:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:52 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 70/1413 objects degraded (4.954%), 12 pgs degraded (PG_DEGRADED) 2024-03-30T13:35:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:52 smithi187 ceph-mon[53604]: osdmap e142: 8 total, 8 up, 8 in 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 7 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:35:53.371 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:35:53.372 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-30T13:35:53.372 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:35:53.372 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:35:53.372 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-03-30T13:35:53.372 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 11 2024-03-30T13:35:53.372 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:35:53.372 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:35:53.711 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "13/23 daemons upgraded", 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading osd daemons", 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:35:53.712 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: pgmap v281: 97 pgs: 4 remapped+peering, 9 peering, 1 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 75 active+clean; 311 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 84 op/s; 71/1164 objects degraded (6.100%); 1.4 MiB/s, 86 keys/s, 20 objects/s recovering 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: failed to encode map e141 with expected crc 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: failed to encode map e141 with expected crc 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: failed to encode map e141 with expected crc 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: failed to encode map e141 with expected crc 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: failed to encode map e142 with expected crc 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: failed to encode map e142 with expected crc 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: failed to encode map e142 with expected crc 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: osdmap e143: 8 total, 8 up, 8 in 2024-03-30T13:35:53.713 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:53 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3784117843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: pgmap v281: 97 pgs: 4 remapped+peering, 9 peering, 1 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 75 active+clean; 311 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 84 op/s; 71/1164 objects degraded (6.100%); 1.4 MiB/s, 86 keys/s, 20 objects/s recovering 2024-03-30T13:35:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: failed to encode map e141 with expected crc 2024-03-30T13:35:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: failed to encode map e141 with expected crc 2024-03-30T13:35:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: failed to encode map e141 with expected crc 2024-03-30T13:35:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: failed to encode map e141 with expected crc 2024-03-30T13:35:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: failed to encode map e142 with expected crc 2024-03-30T13:35:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: failed to encode map e142 with expected crc 2024-03-30T13:35:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: failed to encode map e142 with expected crc 2024-03-30T13:35:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: osdmap e143: 8 total, 8 up, 8 in 2024-03-30T13:35:54.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:53 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3784117843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:35:54.121 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 71/1164 objects degraded (6.100%), 8 pgs degraded 2024-03-30T13:35:54.121 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:35:54.121 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:35:54.121 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 71/1164 objects degraded (6.100%), 8 pgs degraded 2024-03-30T13:35:54.121 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-03-30T13:35:54.121 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-03-30T13:35:54.121 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-03-30T13:35:54.121 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1a is active+undersized+degraded, acting [4,2] 2024-03-30T13:35:54.122 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.a is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-03-30T13:35:54.122 INFO:teuthology.orchestra.run.smithi159.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-03-30T13:35:54.122 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-03-30T13:35:54.122 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-03-30T13:35:55.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:54 smithi159 ceph-mon[67002]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:55.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:54 smithi159 ceph-mon[67002]: failed to encode map e143 with expected crc 2024-03-30T13:35:55.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:54 smithi159 ceph-mon[67002]: osdmap e144: 8 total, 8 up, 8 in 2024-03-30T13:35:55.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:54 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1650894078' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:35:55.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:54 smithi187 ceph-mon[53604]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:55.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:54 smithi187 ceph-mon[53604]: failed to encode map e143 with expected crc 2024-03-30T13:35:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:54 smithi187 ceph-mon[53604]: osdmap e144: 8 total, 8 up, 8 in 2024-03-30T13:35:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:54 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1650894078' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:35:56.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:55 smithi159 ceph-mon[67002]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:56.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:55 smithi159 ceph-mon[67002]: pgmap v285: 97 pgs: 3 remapped+peering, 1 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 253 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 59 KiB/s wr, 74 op/s; 43/630 objects degraded (6.825%); 5.7 MiB/s, 86 keys/s, 63 objects/s recovering 2024-03-30T13:35:56.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:55 smithi159 ceph-mon[67002]: failed to encode map e144 with expected crc 2024-03-30T13:35:56.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:55 smithi159 ceph-mon[67002]: failed to encode map e144 with expected crc 2024-03-30T13:35:56.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:55 smithi159 ceph-mon[67002]: failed to encode map e144 with expected crc 2024-03-30T13:35:56.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:55 smithi159 ceph-mon[67002]: failed to encode map e144 with expected crc 2024-03-30T13:35:56.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:55 smithi159 ceph-mon[67002]: osdmap e145: 8 total, 8 up, 8 in 2024-03-30T13:35:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:55 smithi187 ceph-mon[53604]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:35:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:55 smithi187 ceph-mon[53604]: pgmap v285: 97 pgs: 3 remapped+peering, 1 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 253 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 59 KiB/s wr, 74 op/s; 43/630 objects degraded (6.825%); 5.7 MiB/s, 86 keys/s, 63 objects/s recovering 2024-03-30T13:35:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:55 smithi187 ceph-mon[53604]: failed to encode map e144 with expected crc 2024-03-30T13:35:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:55 smithi187 ceph-mon[53604]: failed to encode map e144 with expected crc 2024-03-30T13:35:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:55 smithi187 ceph-mon[53604]: failed to encode map e144 with expected crc 2024-03-30T13:35:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:55 smithi187 ceph-mon[53604]: failed to encode map e144 with expected crc 2024-03-30T13:35:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:55 smithi187 ceph-mon[53604]: osdmap e145: 8 total, 8 up, 8 in 2024-03-30T13:35:57.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:56 smithi159 ceph-mon[67002]: failed to encode map e145 with expected crc 2024-03-30T13:35:57.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:56 smithi159 ceph-mon[67002]: failed to encode map e145 with expected crc 2024-03-30T13:35:57.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:56 smithi187 ceph-mon[53604]: failed to encode map e145 with expected crc 2024-03-30T13:35:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:56 smithi187 ceph-mon[53604]: failed to encode map e145 with expected crc 2024-03-30T13:35:58.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:57 smithi159 ceph-mon[67002]: pgmap v288: 97 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 229 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 11 KiB/s wr, 64 op/s; 43/438 objects degraded (9.817%); 5.4 MiB/s, 59 objects/s recovering 2024-03-30T13:35:58.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:57 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 43/438 objects degraded (9.817%), 1 pg degraded (PG_DEGRADED) 2024-03-30T13:35:58.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:57 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:58.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:57 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:35:58.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:57 smithi187 ceph-mon[53604]: pgmap v288: 97 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 229 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 11 KiB/s wr, 64 op/s; 43/438 objects degraded (9.817%); 5.4 MiB/s, 59 objects/s recovering 2024-03-30T13:35:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:57 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 43/438 objects degraded (9.817%), 1 pg degraded (PG_DEGRADED) 2024-03-30T13:35:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:57 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:35:58.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:57 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:35:59.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:58 smithi159 ceph-mon[67002]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/438 objects degraded (9.817%), 1 pg degraded) 2024-03-30T13:35:59.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:58 smithi187 ceph-mon[53604]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/438 objects degraded (9.817%), 1 pg degraded) 2024-03-30T13:36:00.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:35:59 smithi159 ceph-mon[67002]: pgmap v289: 97 pgs: 1 peering, 1 active+undersized+remapped, 95 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 34 KiB/s wr, 42 op/s; 916 KiB/s, 42 objects/s recovering 2024-03-30T13:36:00.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:35:59 smithi187 ceph-mon[53604]: pgmap v289: 97 pgs: 1 peering, 1 active+undersized+remapped, 95 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 34 KiB/s wr, 42 op/s; 916 KiB/s, 42 objects/s recovering 2024-03-30T13:36:01.843 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:01 smithi187 ceph-mon[53604]: pgmap v290: 97 pgs: 97 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 895 B/s rd, 29 KiB/s wr, 26 op/s; 783 KiB/s, 36 objects/s recovering 2024-03-30T13:36:01.843 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:01 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-30T13:36:02.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:01 smithi159 ceph-mon[67002]: pgmap v290: 97 pgs: 97 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 895 B/s rd, 29 KiB/s wr, 26 op/s; 783 KiB/s, 36 objects/s recovering 2024-03-30T13:36:02.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:01 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-30T13:36:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:02 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-30T13:36:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:02 smithi187 ceph-mon[53604]: Upgrade: osd.7 is safe to restart 2024-03-30T13:36:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-30T13:36:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:03.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:02 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-30T13:36:03.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:02 smithi159 ceph-mon[67002]: Upgrade: osd.7 is safe to restart 2024-03-30T13:36:03.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:03.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-30T13:36:03.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:03.757 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:03 smithi187 ceph-mon[53604]: pgmap v291: 97 pgs: 97 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 25 KiB/s wr, 9 op/s; 512 KiB/s, 13 objects/s recovering 2024-03-30T13:36:03.758 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:03 smithi187 ceph-mon[53604]: Upgrade: Updating osd.7 2024-03-30T13:36:03.758 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:03 smithi187 ceph-mon[53604]: Deploying daemon osd.7 on smithi187 2024-03-30T13:36:04.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:03 smithi159 ceph-mon[67002]: pgmap v291: 97 pgs: 97 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 25 KiB/s wr, 9 op/s; 512 KiB/s, 13 objects/s recovering 2024-03-30T13:36:04.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:03 smithi159 ceph-mon[67002]: Upgrade: Updating osd.7 2024-03-30T13:36:04.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:03 smithi159 ceph-mon[67002]: Deploying daemon osd.7 on smithi187 2024-03-30T13:36:05.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:05 smithi187 ceph-mon[53604]: pgmap v292: 97 pgs: 97 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 622 B/s rd, 20 KiB/s wr, 8 op/s; 415 KiB/s, 10 objects/s recovering 2024-03-30T13:36:06.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:05 smithi159 ceph-mon[67002]: pgmap v292: 97 pgs: 97 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 622 B/s rd, 20 KiB/s wr, 8 op/s; 415 KiB/s, 10 objects/s recovering 2024-03-30T13:36:06.196 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:05 smithi187 systemd[1]: Stopping Ceph osd.7 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:36:06.521 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:06 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[40797]: 2024-03-30T13:36:06.193+0000 7fc838a60700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:36:06.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:06 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[40797]: 2024-03-30T13:36:06.193+0000 7fc838a60700 -1 osd.7 145 *** Got signal Terminated *** 2024-03-30T13:36:06.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:06 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[40797]: 2024-03-30T13:36:06.193+0000 7fc838a60700 -1 osd.7 145 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:36:07.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:06 smithi159 ceph-mon[67002]: osd.7 marked itself down and dead 2024-03-30T13:36:07.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:06 smithi187 ceph-mon[53604]: osd.7 marked itself down and dead 2024-03-30T13:36:07.579 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:07 smithi187 podman[64838]: 2024-03-30 13:36:07.297328128 +0000 UTC m=+1.222098446 container died 51b657e42788c8a12a53c8af301c614f8b64d48f4a1065c9401fc14b63134aab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, GIT_CLEAN=True, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., version=8, io.buildah.version=1.19.8, name=centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, release=754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-03-30T13:36:07.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:07 smithi187 ceph-mon[53604]: pgmap v293: 97 pgs: 97 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 565 B/s rd, 19 KiB/s wr, 1 op/s; 0 B/s, 3 objects/s recovering 2024-03-30T13:36:07.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:07 smithi187 ceph-mon[53604]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:36:07.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:07 smithi187 ceph-mon[53604]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-03-30T13:36:07.921 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:07 smithi187 ceph-mon[53604]: osdmap e146: 8 total, 7 up, 8 in 2024-03-30T13:36:07.922 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:07.922 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:07 smithi187 podman[64838]: 2024-03-30 13:36:07.677832579 +0000 UTC m=+1.602602889 container cleanup 51b657e42788c8a12a53c8af301c614f8b64d48f4a1065c9401fc14b63134aab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, architecture=x86_64, version=8, GIT_BRANCH=HEAD, vcs-type=git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870) 2024-03-30T13:36:07.922 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:07 smithi187 bash[64838]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7 2024-03-30T13:36:08.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:07 smithi159 ceph-mon[67002]: pgmap v293: 97 pgs: 97 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 565 B/s rd, 19 KiB/s wr, 1 op/s; 0 B/s, 3 objects/s recovering 2024-03-30T13:36:08.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:07 smithi159 ceph-mon[67002]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-30T13:36:08.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:07 smithi159 ceph-mon[67002]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-03-30T13:36:08.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:07 smithi159 ceph-mon[67002]: osdmap e146: 8 total, 7 up, 8 in 2024-03-30T13:36:08.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:08.233 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:07 smithi187 podman[64856]: 2024-03-30 13:36:07.92062899 +0000 UTC m=+0.620209546 container remove 51b657e42788c8a12a53c8af301c614f8b64d48f4a1065c9401fc14b63134aab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, ceph=True, distribution-scope=public, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-30T13:36:08.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:08 smithi187 podman[64947]: 2024-03-30 13:36:08.135081427 +0000 UTC m=+0.017845855 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:36:08.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:08 smithi187 podman[64947]: 2024-03-30 13:36:08.238353378 +0000 UTC m=+0.121117824 container create 5566f30c9123eedb18ecec3ac7a8492324734e1e15d29b31d2615a8454bf3817 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD) 2024-03-30T13:36:08.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:08 smithi187 podman[64947]: 2024-03-30 13:36:08.45779108 +0000 UTC m=+0.340555494 container init 5566f30c9123eedb18ecec3ac7a8492324734e1e15d29b31d2615a8454bf3817 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS) 2024-03-30T13:36:08.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:08 smithi187 podman[64947]: 2024-03-30 13:36:08.461157426 +0000 UTC m=+0.343921833 container start 5566f30c9123eedb18ecec3ac7a8492324734e1e15d29b31d2615a8454bf3817 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True) 2024-03-30T13:36:09.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:08 smithi159 ceph-mon[67002]: osdmap e147: 8 total, 7 up, 8 in 2024-03-30T13:36:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:08 smithi187 ceph-mon[53604]: osdmap e147: 8 total, 7 up, 8 in 2024-03-30T13:36:09.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:08 smithi187 podman[64947]: 2024-03-30 13:36:08.574987772 +0000 UTC m=+0.457752193 container attach 5566f30c9123eedb18ecec3ac7a8492324734e1e15d29b31d2615a8454bf3817 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-03-30T13:36:09.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:08 smithi187 podman[64947]: 2024-03-30 13:36:08.756694898 +0000 UTC m=+0.639459318 container died 5566f30c9123eedb18ecec3ac7a8492324734e1e15d29b31d2615a8454bf3817 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, RELEASE=squid-4e50bbc, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-03-30T13:36:09.665 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:09 smithi187 podman[64979]: 2024-03-30 13:36:09.336785425 +0000 UTC m=+0.575615991 container remove 5566f30c9123eedb18ecec3ac7a8492324734e1e15d29b31d2615a8454bf3817 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-03-30T13:36:09.665 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:09 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.7.service: Deactivated successfully. 2024-03-30T13:36:09.665 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:09 smithi187 systemd[1]: Stopped Ceph osd.7 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:36:09.665 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:09 smithi187 systemd[1]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.7.service: Consumed 7min 41.161s CPU time. 2024-03-30T13:36:09.665 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:09 smithi187 systemd[1]: Starting Ceph osd.7 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:36:10.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:09 smithi159 ceph-mon[67002]: pgmap v296: 97 pgs: 13 peering, 9 stale+active+clean, 75 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 383 B/s wr, 0 op/s 2024-03-30T13:36:10.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:09 smithi159 ceph-mon[67002]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-03-30T13:36:10.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:09 smithi187 ceph-mon[53604]: pgmap v296: 97 pgs: 13 peering, 9 stale+active+clean, 75 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 383 B/s wr, 0 op/s 2024-03-30T13:36:10.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:09 smithi187 ceph-mon[53604]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-03-30T13:36:10.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:09 smithi187 podman[65112]: 2024-03-30 13:36:09.779430978 +0000 UTC m=+0.110397793 container create 0d47dcbe9acc49d2c22e95cf60d7a27a78c34ba28d87eef67fe6685696f74c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:36:10.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:09 smithi187 podman[65112]: 2024-03-30 13:36:09.687721042 +0000 UTC m=+0.018687870 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:36:10.491 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 podman[65112]: 2024-03-30 13:36:10.042622156 +0000 UTC m=+0.373588973 container init 0d47dcbe9acc49d2c22e95cf60d7a27a78c34ba28d87eef67fe6685696f74c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate, org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-03-30T13:36:10.491 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 podman[65112]: 2024-03-30 13:36:10.04582516 +0000 UTC m=+0.376791977 container start 0d47dcbe9acc49d2c22e95cf60d7a27a78c34ba28d87eef67fe6685696f74c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-4e50bbc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5) 2024-03-30T13:36:10.491 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 podman[65112]: 2024-03-30 13:36:10.140596875 +0000 UTC m=+0.471563706 container attach 0d47dcbe9acc49d2c22e95cf60d7a27a78c34ba28d87eef67fe6685696f74c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-03-30T13:36:10.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-mon[53604]: pgmap v297: 97 pgs: 9 active+undersized, 8 active+undersized+degraded, 13 peering, 6 stale+active+clean, 61 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30/426 objects degraded (7.042%) 2024-03-30T13:36:10.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-mon[53604]: Health check failed: Degraded data redundancy: 30/426 objects degraded (7.042%), 8 pgs degraded (PG_DEGRADED) 2024-03-30T13:36:10.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate[65127]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-30T13:36:10.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 bash[65112]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-30T13:36:10.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 bash[65112]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-30T13:36:10.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 bash[65112]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 bash[65112]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 bash[65112]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 bash[65112]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 bash[65112]: --> ceph-volume raw activate successful for osd ID: 7 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate[65127]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate[65127]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate[65127]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate[65127]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate[65127]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate[65127]: --> ceph-volume raw activate successful for osd ID: 7 2024-03-30T13:36:10.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:10 smithi187 podman[65112]: 2024-03-30 13:36:10.514695204 +0000 UTC m=+0.845662015 container died 0d47dcbe9acc49d2c22e95cf60d7a27a78c34ba28d87eef67fe6685696f74c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-03-30T13:36:11.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:10 smithi159 ceph-mon[67002]: pgmap v297: 97 pgs: 9 active+undersized, 8 active+undersized+degraded, 13 peering, 6 stale+active+clean, 61 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30/426 objects degraded (7.042%) 2024-03-30T13:36:11.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:10 smithi159 ceph-mon[67002]: Health check failed: Degraded data redundancy: 30/426 objects degraded (7.042%), 8 pgs degraded (PG_DEGRADED) 2024-03-30T13:36:11.258 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 podman[65244]: 2024-03-30 13:36:11.044348069 +0000 UTC m=+0.520981716 container remove 0d47dcbe9acc49d2c22e95cf60d7a27a78c34ba28d87eef67fe6685696f74c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-30T13:36:11.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 podman[65307]: 2024-03-30 13:36:11.158190651 +0000 UTC m=+0.017926119 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:36:11.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 podman[65307]: 2024-03-30 13:36:11.26188936 +0000 UTC m=+0.121624853 container create b2f37650410f8250af0355f5104156c21bfae155139c61b34d4cf68b856ca102 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:36:11.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 podman[65307]: 2024-03-30 13:36:11.447706733 +0000 UTC m=+0.307442184 container init b2f37650410f8250af0355f5104156c21bfae155139c61b34d4cf68b856ca102 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:36:11.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 podman[65307]: 2024-03-30 13:36:11.451938686 +0000 UTC m=+0.311674146 container start b2f37650410f8250af0355f5104156c21bfae155139c61b34d4cf68b856ca102 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:36:11.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 ceph-osd[65325]: -- 172.21.15.187:0/2550982560 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55def7c0f680 con 0x55def7bd9000 2024-03-30T13:36:11.992 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 bash[65307]: b2f37650410f8250af0355f5104156c21bfae155139c61b34d4cf68b856ca102 2024-03-30T13:36:11.992 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 systemd[1]: Started Ceph osd.7 for 9058771a-ee97-11ee-b647-cb9ed24678a4. 2024-03-30T13:36:12.271 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:11 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:36:11.990+0000 7f227901d7c0 -1 Falling back to public interface 2024-03-30T13:36:12.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:12.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:12.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:12.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:12.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:36:13.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:13.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:13.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:13.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:13.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:36:14.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:13 smithi159 ceph-mon[67002]: pgmap v298: 97 pgs: 16 active+undersized, 13 active+undersized+degraded, 13 peering, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-03-30T13:36:14.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:13 smithi187 ceph-mon[53604]: pgmap v298: 97 pgs: 16 active+undersized, 13 active+undersized+degraded, 13 peering, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-03-30T13:36:14.920 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:14 smithi187 ceph-mon[53604]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-03-30T13:36:15.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:14 smithi159 ceph-mon[67002]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-03-30T13:36:16.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:15 smithi159 ceph-mon[67002]: pgmap v299: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 72/426 objects degraded (16.901%) 2024-03-30T13:36:16.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:15 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:16.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:15 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:16.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:15 smithi187 ceph-mon[53604]: pgmap v299: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 72/426 objects degraded (16.901%) 2024-03-30T13:36:16.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:15 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:16.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:15 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:17 smithi187 ceph-mon[53604]: pgmap v300: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 72/426 objects degraded (16.901%) 2024-03-30T13:36:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:17 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:17 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:17.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:17 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 72/426 objects degraded (16.901%), 21 pgs degraded (PG_DEGRADED) 2024-03-30T13:36:17.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:17 smithi159 ceph-mon[67002]: pgmap v300: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 72/426 objects degraded (16.901%) 2024-03-30T13:36:17.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:17 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:17.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:17 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:17.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:17 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 72/426 objects degraded (16.901%), 21 pgs degraded (PG_DEGRADED) 2024-03-30T13:36:19.582 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:19 smithi187 ceph-mon[53604]: pgmap v301: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 72/426 objects degraded (16.901%) 2024-03-30T13:36:19.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:19 smithi159 ceph-mon[67002]: pgmap v301: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 72/426 objects degraded (16.901%) 2024-03-30T13:36:20.021 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:36:19.578+0000 7f227901d7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-03-30T13:36:21.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:21.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:36:21.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: pgmap v302: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 72/426 objects degraded (16.901%) 2024-03-30T13:36:21.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: pgmap v303: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s; 72/426 objects degraded (16.901%) 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all osd 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-03-30T13:36:21.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: Upgrade: Setting require_osd_release to 19 squid 2024-03-30T13:36:21.005 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: pgmap v302: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 72/426 objects degraded (16.901%) 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: pgmap v303: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s; 72/426 objects degraded (16.901%) 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all osd 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-03-30T13:36:21.023 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-03-30T13:36:21.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-03-30T13:36:21.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-03-30T13:36:21.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-03-30T13:36:21.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-03-30T13:36:21.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-03-30T13:36:21.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: Upgrade: Setting require_osd_release to 19 squid 2024-03-30T13:36:21.024 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-03-30T13:36:21.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:21 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:36:21.202+0000 7f227901d7c0 -1 osd.7 145 log_to_monitors true 2024-03-30T13:36:21.915 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:21 smithi159 ceph-mon[67002]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-03-30T13:36:21.915 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-03-30T13:36:21.915 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:21 smithi159 ceph-mon[67002]: osdmap e148: 8 total, 7 up, 8 in 2024-03-30T13:36:21.915 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi159.jwwwqm"]}]: dispatch 2024-03-30T13:36:21.915 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:21 smithi159 ceph-mon[67002]: Upgrade: It appears safe to stop mds.foofs.smithi159.jwwwqm 2024-03-30T13:36:21.915 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:21 smithi159 ceph-mon[67002]: from='osd.7 [v2:172.21.15.187:6824/2107153462,v1:172.21.15.187:6825/2107153462]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-30T13:36:21.915 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:21 smithi159 ceph-mon[67002]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-30T13:36:22.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:21 smithi187 ceph-mon[53604]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-03-30T13:36:22.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-03-30T13:36:22.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:21 smithi187 ceph-mon[53604]: osdmap e148: 8 total, 7 up, 8 in 2024-03-30T13:36:22.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi159.jwwwqm"]}]: dispatch 2024-03-30T13:36:22.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:21 smithi187 ceph-mon[53604]: Upgrade: It appears safe to stop mds.foofs.smithi159.jwwwqm 2024-03-30T13:36:22.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:21 smithi187 ceph-mon[53604]: from='osd.7 [v2:172.21.15.187:6824/2107153462,v1:172.21.15.187:6825/2107153462]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-30T13:36:22.053 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:21 smithi187 ceph-mon[53604]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-30T13:36:22.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:36:22 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:36:22.050+0000 7f226a475700 -1 osd.7 145 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-30T13:36:23.355 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: pgmap v305: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 258 B/s rd, 0 op/s; 72/426 objects degraded (16.901%) 2024-03-30T13:36:23.355 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-30T13:36:23.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: osdmap e149: 8 total, 7 up, 8 in 2024-03-30T13:36:23.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: from='osd.7 [v2:172.21.15.187:6824/2107153462,v1:172.21.15.187:6825/2107153462]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:36:23.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:36:23.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: Upgrade: Updating mds.foofs.smithi159.jwwwqm 2024-03-30T13:36:23.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:23.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi159.jwwwqm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:36:23.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:23.356 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:23 smithi159 ceph-mon[67002]: Deploying daemon mds.foofs.smithi159.jwwwqm on smithi159 2024-03-30T13:36:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: pgmap v305: 97 pgs: 21 active+undersized, 21 active+undersized+degraded, 55 active+clean; 224 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 258 B/s rd, 0 op/s; 72/426 objects degraded (16.901%) 2024-03-30T13:36:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-30T13:36:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: osdmap e149: 8 total, 7 up, 8 in 2024-03-30T13:36:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: from='osd.7 [v2:172.21.15.187:6824/2107153462,v1:172.21.15.187:6825/2107153462]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:36:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-30T13:36:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: Upgrade: Updating mds.foofs.smithi159.jwwwqm 2024-03-30T13:36:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:23.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi159.jwwwqm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:36:23.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:23.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:23 smithi187 ceph-mon[53604]: Deploying daemon mds.foofs.smithi159.jwwwqm on smithi159 2024-03-30T13:36:24.192 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:24 smithi159 ceph-mon[67002]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:36:24.192 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:24 smithi159 ceph-mon[67002]: osd.7 [v2:172.21.15.187:6824/2107153462,v1:172.21.15.187:6825/2107153462] boot 2024-03-30T13:36:24.192 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:24 smithi159 ceph-mon[67002]: osdmap e150: 8 total, 8 up, 8 in 2024-03-30T13:36:24.192 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:24 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:36:24.192 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:24 smithi159 ceph-mon[67002]: osdmap e151: 8 total, 8 up, 8 in 2024-03-30T13:36:24.509 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:36:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:24 smithi187 ceph-mon[53604]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-30T13:36:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:24 smithi187 ceph-mon[53604]: osd.7 [v2:172.21.15.187:6824/2107153462,v1:172.21.15.187:6825/2107153462] boot 2024-03-30T13:36:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:24 smithi187 ceph-mon[53604]: osdmap e150: 8 total, 8 up, 8 in 2024-03-30T13:36:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:24 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-30T13:36:24.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:24 smithi187 ceph-mon[53604]: osdmap e151: 8 total, 8 up, 8 in 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:24 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[66998]: 2024-03-30T13:36:24.843+0000 7ff76b891700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: pgmap v308: 97 pgs: 7 peering, 18 active+undersized, 17 active+undersized+degraded, 55 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 56/426 objects degraded (13.146%) 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: Health check update: Degraded data redundancy: 56/426 objects degraded (13.146%), 17 pgs degraded (PG_DEGRADED) 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: osdmap e152: 8 total, 8 up, 8 in 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: Standby daemon mds.foofs.smithi187.bkvdqd assigned to filesystem foofs as rank 0 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-30T13:36:25.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-30T13:36:25.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-30T13:36:25.254 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:25 smithi159 ceph-mon[67002]: fsmap foofs:1/1 {0=foofs.smithi187.bkvdqd=up:replay} 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (9m) 2m ago 19m 16.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (5m) 2m ago 19m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (5m) 10s ago 18m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 2m ago 18m - - 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (13m) 2m ago 13m 38.5M - 17.2.0 e1d6a67b021e ea82c114a357 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (13m) 10s ago 13m 17.9M - 17.2.0 e1d6a67b021e 847e0818713c 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (8m) 2m ago 20m 507M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (7m) 10s ago 18m 452M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (7m) 2m ago 20m 49.7M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:36:25.264 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (6m) 10s ago 18m 45.5M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (12m) 2m ago 12m 283M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (12m) 10s ago 12m 86.6M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (9m) 2m ago 18m 13.0M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (8m) 10s ago 18m 18.6M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (5m) 2m ago 17m 356M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (4m) 2m ago 17m 277M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (3m) 2m ago 16m 241M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (2m) 2m ago 16m 12.0M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (2m) 10s ago 15m 338M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (87s) 10s ago 15m 303M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (49s) 10s ago 15m 253M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e4f91e53cf7d 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (14s) 10s ago 14m 15.0M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 b2f37650410f 2024-03-30T13:36:25.265 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (7m) 2m ago 17m 63.2M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: pgmap v308: 97 pgs: 7 peering, 18 active+undersized, 17 active+undersized+degraded, 55 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 56/426 objects degraded (13.146%) 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: Health check update: Degraded data redundancy: 56/426 objects degraded (13.146%), 17 pgs degraded (PG_DEGRADED) 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: osdmap e152: 8 total, 8 up, 8 in 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: Standby daemon mds.foofs.smithi187.bkvdqd assigned to filesystem foofs as rank 0 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-30T13:36:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-30T13:36:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-30T13:36:25.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:25 smithi187 ceph-mon[53604]: fsmap foofs:1/1 {0=foofs.smithi187.bkvdqd=up:replay} 2024-03-30T13:36:25.680 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:36:25.680 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 8 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 12 2024-03-30T13:36:25.681 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:36:25.682 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "mon", 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "osd", 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "crash" 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "14/23 daemons upgraded", 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading mds daemons", 2024-03-30T13:36:26.021 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:36:26.022 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:36:26.424 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 filesystem is degraded; insufficient standby MDS daemons available; Degraded data redundancy: 36/426 objects degraded (8.451%), 11 pgs degraded 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: fs foofs is degraded 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: have 0; want 1 more 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 36/426 objects degraded (8.451%), 11 pgs degraded 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-03-30T13:36:26.425 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-03-30T13:36:26.426 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-03-30T13:36:26.426 INFO:teuthology.orchestra.run.smithi159.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-03-30T13:36:26.426 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-03-30T13:36:26.426 INFO:teuthology.orchestra.run.smithi159.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-03-30T13:36:26.426 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:26 smithi159 ceph-mon[67002]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:26.426 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:26 smithi159 ceph-mon[67002]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:26.426 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:26 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3769262954' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:26.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:26 smithi187 ceph-mon[53604]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:26.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:26 smithi187 ceph-mon[53604]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:26.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:26 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3769262954' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:27.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:27 smithi159 ceph-mon[67002]: pgmap v311: 97 pgs: 7 peering, 14 active+undersized, 11 active+undersized+degraded, 65 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2 op/s; 36/426 objects degraded (8.451%) 2024-03-30T13:36:27.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:27 smithi159 ceph-mon[67002]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:27.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:27 smithi159 ceph-mon[67002]: reconnect by client.24361 172.21.15.187:0/2657843257 after 0 2024-03-30T13:36:27.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:27 smithi159 ceph-mon[67002]: reconnect by client.14530 172.21.15.159:0/72489277 after 0 2024-03-30T13:36:27.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:27 smithi159 ceph-mon[67002]: mds.? [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] up:reconnect 2024-03-30T13:36:27.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:27 smithi159 ceph-mon[67002]: fsmap foofs:1/1 {0=foofs.smithi187.bkvdqd=up:reconnect} 2024-03-30T13:36:27.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:27 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1961623205' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:36:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:27 smithi187 ceph-mon[53604]: pgmap v311: 97 pgs: 7 peering, 14 active+undersized, 11 active+undersized+degraded, 65 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2 op/s; 36/426 objects degraded (8.451%) 2024-03-30T13:36:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:27 smithi187 ceph-mon[53604]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:27 smithi187 ceph-mon[53604]: reconnect by client.24361 172.21.15.187:0/2657843257 after 0 2024-03-30T13:36:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:27 smithi187 ceph-mon[53604]: reconnect by client.14530 172.21.15.159:0/72489277 after 0 2024-03-30T13:36:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:27 smithi187 ceph-mon[53604]: mds.? [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] up:reconnect 2024-03-30T13:36:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:27 smithi187 ceph-mon[53604]: fsmap foofs:1/1 {0=foofs.smithi187.bkvdqd=up:reconnect} 2024-03-30T13:36:27.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:27 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1961623205' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:36:28.409 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:28 smithi187 ceph-mon[53604]: mds.? [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] up:rejoin 2024-03-30T13:36:28.409 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:28 smithi187 ceph-mon[53604]: fsmap foofs:1/1 {0=foofs.smithi187.bkvdqd=up:rejoin} 2024-03-30T13:36:28.410 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:28 smithi187 ceph-mon[53604]: daemon mds.foofs.smithi187.bkvdqd is now active in filesystem foofs as rank 0 2024-03-30T13:36:28.410 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:28.410 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:36:28.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:28 smithi159 ceph-mon[67002]: mds.? [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] up:rejoin 2024-03-30T13:36:28.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:28 smithi159 ceph-mon[67002]: fsmap foofs:1/1 {0=foofs.smithi187.bkvdqd=up:rejoin} 2024-03-30T13:36:28.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:28 smithi159 ceph-mon[67002]: daemon mds.foofs.smithi187.bkvdqd is now active in filesystem foofs as rank 0 2024-03-30T13:36:28.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:28.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:36:29.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:29 smithi159 ceph-mon[67002]: pgmap v312: 97 pgs: 7 peering, 90 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 8 op/s 2024-03-30T13:36:29.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:29 smithi159 ceph-mon[67002]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-30T13:36:29.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:29 smithi159 ceph-mon[67002]: mds.? [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] up:active 2024-03-30T13:36:29.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:29 smithi159 ceph-mon[67002]: fsmap foofs:1 {0=foofs.smithi187.bkvdqd=up:active} 2024-03-30T13:36:29.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:36:29.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:29 smithi159 ceph-mon[67002]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/426 objects degraded (8.451%), 11 pgs degraded) 2024-03-30T13:36:29.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:29 smithi187 ceph-mon[53604]: pgmap v312: 97 pgs: 7 peering, 90 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 8 op/s 2024-03-30T13:36:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:29 smithi187 ceph-mon[53604]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-30T13:36:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:29 smithi187 ceph-mon[53604]: mds.? [v2:172.21.15.187:6832/1184366614,v1:172.21.15.187:6833/1184366614] up:active 2024-03-30T13:36:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:29 smithi187 ceph-mon[53604]: fsmap foofs:1 {0=foofs.smithi187.bkvdqd=up:active} 2024-03-30T13:36:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:36:29.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:29 smithi187 ceph-mon[53604]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/426 objects degraded (8.451%), 11 pgs degraded) 2024-03-30T13:36:31.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:31 smithi159 ceph-mon[67002]: pgmap v313: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 12 op/s 2024-03-30T13:36:31.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:31 smithi187 ceph-mon[53604]: pgmap v313: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 12 op/s 2024-03-30T13:36:33.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:33 smithi159 ceph-mon[67002]: pgmap v314: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 11 KiB/s wr, 15 op/s 2024-03-30T13:36:33.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:33 smithi187 ceph-mon[53604]: pgmap v314: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 11 KiB/s wr, 15 op/s 2024-03-30T13:36:35.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:35 smithi159 ceph-mon[67002]: pgmap v315: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 8.9 KiB/s wr, 14 op/s 2024-03-30T13:36:35.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:35 smithi187 ceph-mon[53604]: pgmap v315: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 8.9 KiB/s wr, 14 op/s 2024-03-30T13:36:37.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:37 smithi159 ceph-mon[67002]: pgmap v316: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 8.0 KiB/s wr, 13 op/s 2024-03-30T13:36:37.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:37 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:37.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:37 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:37.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:37 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:37.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:37 smithi159 ceph-mon[67002]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-30T13:36:37.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:37 smithi159 ceph-mon[67002]: mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772] up:boot 2024-03-30T13:36:37.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:37 smithi159 ceph-mon[67002]: fsmap foofs:1 {0=foofs.smithi187.bkvdqd=up:active} 1 up:standby 2024-03-30T13:36:37.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:37 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:36:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:37 smithi187 ceph-mon[53604]: pgmap v316: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 8.0 KiB/s wr, 13 op/s 2024-03-30T13:36:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:37 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:37 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:37 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:37 smithi187 ceph-mon[53604]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-30T13:36:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:37 smithi187 ceph-mon[53604]: mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772] up:boot 2024-03-30T13:36:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:37 smithi187 ceph-mon[53604]: fsmap foofs:1 {0=foofs.smithi187.bkvdqd=up:active} 1 up:standby 2024-03-30T13:36:37.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:37 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:36:39.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:38 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:39.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:38 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:39.940 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:39 smithi187 ceph-mon[53604]: pgmap v317: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-03-30T13:36:40.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:39 smithi159 ceph-mon[67002]: pgmap v317: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-03-30T13:36:42.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:41 smithi159 ceph-mon[67002]: pgmap v318: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-03-30T13:36:42.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:41 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:42.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:41 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:42.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:41 smithi187 ceph-mon[53604]: pgmap v318: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-03-30T13:36:42.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:41 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:42.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:41 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:43.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:43 smithi159 ceph-mon[67002]: pgmap v319: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 KiB/s rd, 7.4 KiB/s wr, 5 op/s 2024-03-30T13:36:43.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:43.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:43.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:43.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:36:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:43 smithi187 ceph-mon[53604]: pgmap v319: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 KiB/s rd, 7.4 KiB/s wr, 5 op/s 2024-03-30T13:36:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:43.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:36:45.287 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:45 smithi159 ceph-mon[67002]: pgmap v320: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 KiB/s rd, 255 B/s wr, 1 op/s 2024-03-30T13:36:45.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:45 smithi187 ceph-mon[53604]: pgmap v320: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 KiB/s rd, 255 B/s wr, 1 op/s 2024-03-30T13:36:47.105 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: pgmap v321: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-03-30T13:36:47.105 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: Detected new or changed devices on smithi159 2024-03-30T13:36:47.105 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:47.105 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:47.105 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: pgmap v322: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 199 B/s rd, 199 B/s wr, 0 op/s 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: pgmap v323: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 247 B/s rd, 247 B/s wr, 0 op/s 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:47.106 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi187.bkvdqd"]}]: dispatch 2024-03-30T13:36:47.107 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:47 smithi187 ceph-mon[53604]: Upgrade: It appears safe to stop mds.foofs.smithi187.bkvdqd 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: pgmap v321: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: Detected new or changed devices on smithi159 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: pgmap v322: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 199 B/s rd, 199 B/s wr, 0 op/s 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: pgmap v323: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 247 B/s rd, 247 B/s wr, 0 op/s 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:47.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:47.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:47.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:47.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi187.bkvdqd"]}]: dispatch 2024-03-30T13:36:47.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:47 smithi159 ceph-mon[67002]: Upgrade: It appears safe to stop mds.foofs.smithi187.bkvdqd 2024-03-30T13:36:48.989 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:48 smithi187 ceph-mon[53604]: Upgrade: Updating mds.foofs.smithi187.bkvdqd 2024-03-30T13:36:48.990 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:48 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:48.990 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:48 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi187.bkvdqd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:36:48.990 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:48 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:48.990 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:48 smithi187 ceph-mon[53604]: Deploying daemon mds.foofs.smithi187.bkvdqd on smithi187 2024-03-30T13:36:48.990 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:48 smithi187 ceph-mon[53604]: pgmap v324: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 247 B/s rd, 247 B/s wr, 0 op/s 2024-03-30T13:36:49.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:48 smithi159 ceph-mon[67002]: Upgrade: Updating mds.foofs.smithi187.bkvdqd 2024-03-30T13:36:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:48 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:48 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi187.bkvdqd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-30T13:36:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:48 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:36:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:48 smithi159 ceph-mon[67002]: Deploying daemon mds.foofs.smithi187.bkvdqd on smithi187 2024-03-30T13:36:49.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:48 smithi159 ceph-mon[67002]: pgmap v324: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 247 B/s rd, 247 B/s wr, 0 op/s 2024-03-30T13:36:50.003 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:36:49 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:36:49.771+0000 7f185a128700 -1 bdev(0x562a8c9fa000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-03-30T13:36:50.003 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:36:49 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:36:49.771+0000 7f185b92b700 -1 bdev(0x562a8c9fa000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-03-30T13:36:50.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:49 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[66998]: 2024-03-30T13:36:49.744+0000 7ff76b891700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:36:50.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:49 smithi159 ceph-mon[67002]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-30T13:36:50.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:49 smithi159 ceph-mon[67002]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:36:50.009 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:49 smithi187 ceph-mon[53604]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-30T13:36:50.009 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:49 smithi187 ceph-mon[53604]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-30T13:36:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:50 smithi187 ceph-mon[53604]: osdmap e153: 8 total, 8 up, 8 in 2024-03-30T13:36:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:50 smithi187 ceph-mon[53604]: Standby daemon mds.foofs.smithi159.jwwwqm assigned to filesystem foofs as rank 0 2024-03-30T13:36:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:50 smithi187 ceph-mon[53604]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-30T13:36:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:50 smithi187 ceph-mon[53604]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-30T13:36:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:50 smithi187 ceph-mon[53604]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-30T13:36:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:50 smithi187 ceph-mon[53604]: fsmap foofs:1/1 {0=foofs.smithi159.jwwwqm=up:replay} 2024-03-30T13:36:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:50 smithi187 ceph-mon[53604]: pgmap v326: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 326 B/s rd, 0 op/s 2024-03-30T13:36:51.223 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:50 smithi159 ceph-mon[67002]: osdmap e153: 8 total, 8 up, 8 in 2024-03-30T13:36:51.224 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:50 smithi159 ceph-mon[67002]: Standby daemon mds.foofs.smithi159.jwwwqm assigned to filesystem foofs as rank 0 2024-03-30T13:36:51.224 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:50 smithi159 ceph-mon[67002]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-30T13:36:51.224 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:50 smithi159 ceph-mon[67002]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-30T13:36:51.224 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:50 smithi159 ceph-mon[67002]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-30T13:36:51.224 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:50 smithi159 ceph-mon[67002]: fsmap foofs:1/1 {0=foofs.smithi159.jwwwqm=up:replay} 2024-03-30T13:36:51.224 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:50 smithi159 ceph-mon[67002]: pgmap v326: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 326 B/s rd, 0 op/s 2024-03-30T13:36:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:51 smithi187 ceph-mon[53604]: reconnect by client.14530 172.21.15.159:0/72489277 after 0 2024-03-30T13:36:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:51 smithi187 ceph-mon[53604]: reconnect by client.24361 172.21.15.187:0/2657843257 after 0 2024-03-30T13:36:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:51 smithi187 ceph-mon[53604]: mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772] up:reconnect 2024-03-30T13:36:52.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:51 smithi187 ceph-mon[53604]: fsmap foofs:1/1 {0=foofs.smithi159.jwwwqm=up:reconnect} 2024-03-30T13:36:52.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:51 smithi159 ceph-mon[67002]: reconnect by client.14530 172.21.15.159:0/72489277 after 0 2024-03-30T13:36:52.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:51 smithi159 ceph-mon[67002]: reconnect by client.24361 172.21.15.187:0/2657843257 after 0 2024-03-30T13:36:52.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:51 smithi159 ceph-mon[67002]: mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772] up:reconnect 2024-03-30T13:36:52.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:51 smithi159 ceph-mon[67002]: fsmap foofs:1/1 {0=foofs.smithi159.jwwwqm=up:reconnect} 2024-03-30T13:36:53.123 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:52 smithi187 ceph-mon[53604]: mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772] up:rejoin 2024-03-30T13:36:53.123 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:52 smithi187 ceph-mon[53604]: fsmap foofs:1/1 {0=foofs.smithi159.jwwwqm=up:rejoin} 2024-03-30T13:36:53.123 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:52 smithi187 ceph-mon[53604]: daemon mds.foofs.smithi159.jwwwqm is now active in filesystem foofs as rank 0 2024-03-30T13:36:53.123 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:52 smithi187 ceph-mon[53604]: pgmap v327: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 163 B/s wr, 5 op/s 2024-03-30T13:36:53.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:52 smithi159 ceph-mon[67002]: mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772] up:rejoin 2024-03-30T13:36:53.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:52 smithi159 ceph-mon[67002]: fsmap foofs:1/1 {0=foofs.smithi159.jwwwqm=up:rejoin} 2024-03-30T13:36:53.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:52 smithi159 ceph-mon[67002]: daemon mds.foofs.smithi159.jwwwqm is now active in filesystem foofs as rank 0 2024-03-30T13:36:53.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:52 smithi159 ceph-mon[67002]: pgmap v327: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 163 B/s wr, 5 op/s 2024-03-30T13:36:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:53 smithi187 ceph-mon[53604]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-30T13:36:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:53 smithi187 ceph-mon[53604]: mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772] up:active 2024-03-30T13:36:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:53 smithi187 ceph-mon[53604]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 2024-03-30T13:36:54.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:53 smithi159 ceph-mon[67002]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-30T13:36:54.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:53 smithi159 ceph-mon[67002]: mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772] up:active 2024-03-30T13:36:54.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:53 smithi159 ceph-mon[67002]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 2024-03-30T13:36:55.258 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:54 smithi159 ceph-mon[67002]: pgmap v328: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 127 B/s wr, 9 op/s 2024-03-30T13:36:55.259 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:54 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:55.259 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:54 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:55.259 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:54 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:54 smithi187 ceph-mon[53604]: pgmap v328: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 127 B/s wr, 9 op/s 2024-03-30T13:36:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:54 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:54 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:55.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:54 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:36:56.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:55 smithi187 ceph-mon[53604]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-30T13:36:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:55 smithi187 ceph-mon[53604]: mds.? [v2:172.21.15.187:6832/3052965722,v1:172.21.15.187:6833/3052965722] up:boot 2024-03-30T13:36:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:55 smithi187 ceph-mon[53604]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:36:56.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:36:56.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:55 smithi159 ceph-mon[67002]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-30T13:36:56.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:55 smithi159 ceph-mon[67002]: mds.? [v2:172.21.15.187:6832/3052965722,v1:172.21.15.187:6833/3052965722] up:boot 2024-03-30T13:36:56.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:55 smithi159 ceph-mon[67002]: fsmap foofs:1 {0=foofs.smithi159.jwwwqm=up:active} 1 up:standby 2024-03-30T13:36:56.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:36:56.869 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:36:57.082 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:56 smithi159 ceph-mon[67002]: pgmap v329: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 102 B/s wr, 9 op/s 2024-03-30T13:36:57.104 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:56 smithi187 ceph-mon[53604]: pgmap v329: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 102 B/s wr, 9 op/s 2024-03-30T13:36:57.604 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:36:57.604 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (10m) 16s ago 19m 25.3M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (6m) 16s ago 19m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (6m) 42s ago 18m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 16s ago 19m - - 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (21s) 16s ago 13m 11.2M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 322b0f7af8dd 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 starting - - - - 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (9m) 16s ago 20m 511M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (8m) 42s ago 18m 452M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (7m) 16s ago 21m 58.4M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (7m) 42s ago 18m 45.5M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (13m) 16s ago 13m 263M - 4.0 e1d6a67b021e 4abddc42536b 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (13m) 42s ago 13m 86.6M - 4.0 e1d6a67b021e c371d2c7d01d 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (9m) 16s ago 19m 19.0M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (9m) 42s ago 18m 18.6M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (5m) 16s ago 18m 385M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:36:57.605 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (4m) 16s ago 17m 340M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:36:57.606 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (4m) 16s ago 17m 353M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:36:57.606 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (3m) 16s ago 16m 386M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:36:57.606 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (2m) 42s ago 16m 338M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:36:57.606 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (2m) 42s ago 16m 303M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:36:57.606 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (82s) 42s ago 15m 253M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e4f91e53cf7d 2024-03-30T13:36:57.606 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (46s) 42s ago 15m 15.0M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 b2f37650410f 2024-03-30T13:36:57.606 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (8m) 16s ago 18m 67.7M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:36:58.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:57 smithi159 ceph-mon[67002]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:58.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:57 smithi159 ceph-mon[67002]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:58.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:57 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:36:58.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:57 smithi159 ceph-mon[67002]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:58.039 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:36:58.039 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 8 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 14 2024-03-30T13:36:58.040 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:36:58.041 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:36:58.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:57 smithi187 ceph-mon[53604]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:58.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:57 smithi187 ceph-mon[53604]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:58.049 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:57 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:36:58.050 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:57 smithi187 ceph-mon[53604]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:58.381 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "osd", 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "crash", 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "mds", 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "mon" 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:36:58.382 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "16/23 daemons upgraded", 2024-03-30T13:36:58.383 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading mds daemons", 2024-03-30T13:36:58.383 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:36:58.383 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:36:58.787 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:36:58.787 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:36:58.787 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:36:59.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:59.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:59.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:58 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/203981914' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:59.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:58 smithi159 ceph-mon[67002]: pgmap v330: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.8 KiB/s wr, 14 op/s 2024-03-30T13:36:59.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:58 smithi159 ceph-mon[67002]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:59.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:58 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/3839024336' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:36:59.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:59.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:36:59.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:58 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/203981914' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:36:59.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:58 smithi187 ceph-mon[53604]: pgmap v330: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.8 KiB/s wr, 14 op/s 2024-03-30T13:36:59.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:58 smithi187 ceph-mon[53604]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:36:59.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:58 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/3839024336' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:37:00.252 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:59 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:00.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:36:59 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:00.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:59 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:00.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:36:59 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:00 smithi187 ceph-mon[53604]: pgmap v331: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.5 KiB/s wr, 14 op/s 2024-03-30T13:37:01.224 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:00 smithi159 ceph-mon[67002]: pgmap v331: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.5 KiB/s wr, 14 op/s 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: pgmap v332: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 7.3 KiB/s wr, 12 op/s 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: Detected new or changed devices on smithi187 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: pgmap v333: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 8.5 KiB/s wr, 11 op/s 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: pgmap v334: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 11 KiB/s wr, 8 op/s 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:03.389 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all mds 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi159.jwwwqm"}]': finished 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi187.bkvdqd"}]': finished 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all rgw 2024-03-30T13:37:03.390 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all rbd-mirror 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all ceph-exporter 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all iscsi 2024-03-30T13:37:03.391 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:03 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: pgmap v332: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 7.3 KiB/s wr, 12 op/s 2024-03-30T13:37:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: Detected new or changed devices on smithi187 2024-03-30T13:37:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: pgmap v333: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 8.5 KiB/s wr, 11 op/s 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: pgmap v334: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 11 KiB/s wr, 8 op/s 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all mds 2024-03-30T13:37:03.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi159.jwwwqm"}]: dispatch 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi159.jwwwqm"}]': finished 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi187.bkvdqd"}]: dispatch 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi187.bkvdqd"}]': finished 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all rgw 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all rbd-mirror 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all ceph-exporter 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:03.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:03.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all iscsi 2024-03-30T13:37:03.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:03 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:05.576 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: pgmap v335: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 11 KiB/s wr, 6 op/s 2024-03-30T13:37:05.576 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:05.576 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-30T13:37:05.576 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-30T13:37:05.576 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-30T13:37:05.577 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:05.577 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-30T13:37:05.577 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-30T13:37:05.577 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-30T13:37:05.577 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:05 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: pgmap v335: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 11 KiB/s wr, 6 op/s 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi159.vhnwtw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-30T13:37:05.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:05 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:06.579 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:06 smithi159 ceph-mon[67002]: Upgrade: Updating nfs.foo.0.0.smithi159.vhnwtw (1/2) 2024-03-30T13:37:06.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:06 smithi159 ceph-mon[67002]: Creating key for client.nfs.foo.0.0.smithi159.vhnwtw 2024-03-30T13:37:06.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:06 smithi159 ceph-mon[67002]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-30T13:37:06.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:06 smithi159 ceph-mon[67002]: Rados config object exists: conf-nfs.foo 2024-03-30T13:37:06.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:06 smithi159 ceph-mon[67002]: Creating key for client.nfs.foo.0.0.smithi159.vhnwtw-rgw 2024-03-30T13:37:06.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:06 smithi159 ceph-mon[67002]: Bind address in nfs.foo.0.0.smithi159.vhnwtw's ganesha conf is defaulting to empty 2024-03-30T13:37:06.580 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:06 smithi159 ceph-mon[67002]: Deploying daemon nfs.foo.0.0.smithi159.vhnwtw on smithi159 2024-03-30T13:37:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:06 smithi187 ceph-mon[53604]: Upgrade: Updating nfs.foo.0.0.smithi159.vhnwtw (1/2) 2024-03-30T13:37:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:06 smithi187 ceph-mon[53604]: Creating key for client.nfs.foo.0.0.smithi159.vhnwtw 2024-03-30T13:37:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:06 smithi187 ceph-mon[53604]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-30T13:37:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:06 smithi187 ceph-mon[53604]: Rados config object exists: conf-nfs.foo 2024-03-30T13:37:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:06 smithi187 ceph-mon[53604]: Creating key for client.nfs.foo.0.0.smithi159.vhnwtw-rgw 2024-03-30T13:37:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:06 smithi187 ceph-mon[53604]: Bind address in nfs.foo.0.0.smithi159.vhnwtw's ganesha conf is defaulting to empty 2024-03-30T13:37:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:06 smithi187 ceph-mon[53604]: Deploying daemon nfs.foo.0.0.smithi159.vhnwtw on smithi159 2024-03-30T13:37:07.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:07 smithi159 ceph-mon[67002]: pgmap v336: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 250 B/s rd, 0 op/s 2024-03-30T13:37:07.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:07 smithi187 ceph-mon[53604]: pgmap v336: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 250 B/s rd, 0 op/s 2024-03-30T13:37:09.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:09 smithi159 ceph-mon[67002]: pgmap v337: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 250 B/s rd, 250 B/s wr, 0 op/s 2024-03-30T13:37:09.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:09 smithi159 ceph-mon[67002]: from='mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772]' entity='mds.foofs.smithi159.jwwwqm' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.159:0/72489277"}]: dispatch 2024-03-30T13:37:09.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:09.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:09.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:09 smithi187 ceph-mon[53604]: pgmap v337: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 250 B/s rd, 250 B/s wr, 0 op/s 2024-03-30T13:37:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:09 smithi187 ceph-mon[53604]: from='mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772]' entity='mds.foofs.smithi159.jwwwqm' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.159:0/72489277"}]: dispatch 2024-03-30T13:37:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:09.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:10 smithi187 ceph-mon[53604]: Evicting (and blocklisting) client session 14530 (172.21.15.159:0/72489277) 2024-03-30T13:37:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:10 smithi187 ceph-mon[53604]: from='mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772]' entity='mds.foofs.smithi159.jwwwqm' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.159:0/72489277"}]': finished 2024-03-30T13:37:10.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:10 smithi187 ceph-mon[53604]: osdmap e154: 8 total, 8 up, 8 in 2024-03-30T13:37:10.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:10 smithi159 ceph-mon[67002]: Evicting (and blocklisting) client session 14530 (172.21.15.159:0/72489277) 2024-03-30T13:37:10.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:10 smithi159 ceph-mon[67002]: from='mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772]' entity='mds.foofs.smithi159.jwwwqm' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.159:0/72489277"}]': finished 2024-03-30T13:37:10.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:10 smithi159 ceph-mon[67002]: osdmap e154: 8 total, 8 up, 8 in 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: pgmap v339: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 895 B/s wr, 3 op/s 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-30T13:37:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:11 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:11.590 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: pgmap v339: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 895 B/s wr, 3 op/s 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi187.dwmvfp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-30T13:37:11.591 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:11 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:12.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:12 smithi187 ceph-mon[53604]: Upgrade: Updating nfs.foo.1.0.smithi187.dwmvfp (2/2) 2024-03-30T13:37:12.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:12 smithi187 ceph-mon[53604]: Creating key for client.nfs.foo.1.0.smithi187.dwmvfp 2024-03-30T13:37:12.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:12 smithi187 ceph-mon[53604]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-30T13:37:12.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:12 smithi187 ceph-mon[53604]: Rados config object exists: conf-nfs.foo 2024-03-30T13:37:12.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:12 smithi187 ceph-mon[53604]: Creating key for client.nfs.foo.1.0.smithi187.dwmvfp-rgw 2024-03-30T13:37:12.497 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:12 smithi187 ceph-mon[53604]: Bind address in nfs.foo.1.0.smithi187.dwmvfp's ganesha conf is defaulting to empty 2024-03-30T13:37:12.498 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:12 smithi187 ceph-mon[53604]: Deploying daemon nfs.foo.1.0.smithi187.dwmvfp on smithi187 2024-03-30T13:37:12.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:12 smithi159 ceph-mon[67002]: Upgrade: Updating nfs.foo.1.0.smithi187.dwmvfp (2/2) 2024-03-30T13:37:12.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:12 smithi159 ceph-mon[67002]: Creating key for client.nfs.foo.1.0.smithi187.dwmvfp 2024-03-30T13:37:12.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:12 smithi159 ceph-mon[67002]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-30T13:37:12.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:12 smithi159 ceph-mon[67002]: Rados config object exists: conf-nfs.foo 2024-03-30T13:37:12.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:12 smithi159 ceph-mon[67002]: Creating key for client.nfs.foo.1.0.smithi187.dwmvfp-rgw 2024-03-30T13:37:12.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:12 smithi159 ceph-mon[67002]: Bind address in nfs.foo.1.0.smithi187.dwmvfp's ganesha conf is defaulting to empty 2024-03-30T13:37:12.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:12 smithi159 ceph-mon[67002]: Deploying daemon nfs.foo.1.0.smithi187.dwmvfp on smithi187 2024-03-30T13:37:13.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:13 smithi159 ceph-mon[67002]: pgmap v340: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-30T13:37:13.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:13 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:37:13.711 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:13 smithi187 ceph-mon[53604]: pgmap v340: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-30T13:37:13.711 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:13 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:37:15.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:15 smithi187 ceph-mon[53604]: pgmap v341: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-30T13:37:15.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:15 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:15.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:15 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:15.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:15 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:16.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:15 smithi159 ceph-mon[67002]: pgmap v341: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-30T13:37:16.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:15 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:16.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:15 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:16.004 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:15 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:17.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:17 smithi159 ceph-mon[67002]: pgmap v342: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-03-30T13:37:17.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:17 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:17.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:17 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:18.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:17 smithi187 ceph-mon[53604]: pgmap v342: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-03-30T13:37:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:17 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:18.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:17 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:18.665 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:18 smithi187 ceph-mon[53604]: Evicting (and blocklisting) client session 24361 (172.21.15.187:0/2657843257) 2024-03-30T13:37:18.666 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:18 smithi187 ceph-mon[53604]: from='mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772]' entity='mds.foofs.smithi159.jwwwqm' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.187:0/2657843257"}]: dispatch 2024-03-30T13:37:18.666 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:18.666 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:18 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:19.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:18 smithi159 ceph-mon[67002]: Evicting (and blocklisting) client session 24361 (172.21.15.187:0/2657843257) 2024-03-30T13:37:19.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:18 smithi159 ceph-mon[67002]: from='mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772]' entity='mds.foofs.smithi159.jwwwqm' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.187:0/2657843257"}]: dispatch 2024-03-30T13:37:19.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:19.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:18 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:19.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:19 smithi187 ceph-mon[53604]: pgmap v343: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-03-30T13:37:19.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:19 smithi187 ceph-mon[53604]: from='mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772]' entity='mds.foofs.smithi159.jwwwqm' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.187:0/2657843257"}]': finished 2024-03-30T13:37:19.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:19 smithi187 ceph-mon[53604]: osdmap e155: 8 total, 8 up, 8 in 2024-03-30T13:37:19.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:19 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:19.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:19 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:20.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:19 smithi159 ceph-mon[67002]: pgmap v343: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-03-30T13:37:20.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:19 smithi159 ceph-mon[67002]: from='mds.? [v2:172.21.15.159:6834/2654641772,v1:172.21.15.159:6835/2654641772]' entity='mds.foofs.smithi159.jwwwqm' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.187:0/2657843257"}]': finished 2024-03-30T13:37:20.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:19 smithi159 ceph-mon[67002]: osdmap e155: 8 total, 8 up, 8 in 2024-03-30T13:37:20.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:19 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:20.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:19 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:21.874 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:21 smithi187 ceph-mon[53604]: pgmap v345: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-03-30T13:37:21.874 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:21.874 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:21 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:22.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:21 smithi159 ceph-mon[67002]: pgmap v345: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-03-30T13:37:22.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:22.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:21 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:23.244 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:22 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:23.245 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:22 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:23.245 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:22 smithi159 ceph-mon[67002]: pgmap v346: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-30T13:37:23.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:22 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:23.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:22 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:23.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:22 smithi187 ceph-mon[53604]: pgmap v346: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-30T13:37:25.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: pgmap v347: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-30T13:37:25.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:25.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:37:25.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: pgmap v348: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-03-30T13:37:25.339 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: pgmap v349: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all nfs 2024-03-30T13:37:25.340 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi159.vhnwtw"}]: dispatch 2024-03-30T13:37:25.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi159.vhnwtw"}]': finished 2024-03-30T13:37:25.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi187.dwmvfp"}]: dispatch 2024-03-30T13:37:25.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi187.dwmvfp"}]': finished 2024-03-30T13:37:25.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: Upgrade: Setting container_image for all nvmeof 2024-03-30T13:37:25.341 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: pgmap v347: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-30T13:37:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:25.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: pgmap v348: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: pgmap v349: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all nfs 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi159.vhnwtw"}]: dispatch 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi159.vhnwtw"}]': finished 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi187.dwmvfp"}]: dispatch 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi187.dwmvfp"}]': finished 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: Upgrade: Setting container_image for all nvmeof 2024-03-30T13:37:25.774 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:27.448 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:27 smithi159 ceph-mon[67002]: Upgrade: Updating node-exporter.smithi159 (1/2) 2024-03-30T13:37:27.448 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:27 smithi159 ceph-mon[67002]: Deploying daemon node-exporter.smithi159 on smithi159 2024-03-30T13:37:27.448 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:27 smithi159 ceph-mon[67002]: pgmap v350: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-03-30T13:37:27.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:27 smithi187 ceph-mon[53604]: Upgrade: Updating node-exporter.smithi159 (1/2) 2024-03-30T13:37:27.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:27 smithi187 ceph-mon[53604]: Deploying daemon node-exporter.smithi159 on smithi159 2024-03-30T13:37:27.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:27 smithi187 ceph-mon[53604]: pgmap v350: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-03-30T13:37:28.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:37:28.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:37:29.164 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:37:29.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:29 smithi159 ceph-mon[67002]: pgmap v351: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 507 B/s wr, 0 op/s 2024-03-30T13:37:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:29 smithi187 ceph-mon[53604]: pgmap v351: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 507 B/s wr, 0 op/s 2024-03-30T13:37:30.069 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:37:30.069 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (10m) 10s ago 20m 25.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:37:30.069 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (6m) 10s ago 20m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:37:30.069 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (6m) 12s ago 19m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 10s ago 19m - - 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (54s) 10s ago 14m 34.3M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 322b0f7af8dd 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (35s) 12s ago 14m 13.9M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 88d78b5e9b1d 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (9m) 10s ago 21m 512M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (8m) 12s ago 19m 452M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (8m) 10s ago 21m 60.1M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (7m) 12s ago 19m 47.0M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (21s) 10s ago 14m 57.4M - 5.7 b0c4478f1d83 785c7ee033e2 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (15s) 12s ago 13m 16.9M - 5.7 b0c4478f1d83 db1b13dcf23b 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (10m) 10s ago 19m 21.1M - 1.3.1 1dbe0e931976 f0171d1bbe19 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (9m) 12s ago 19m 20.2M - 1.3.1 1dbe0e931976 e3c90069f188 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (6m) 10s ago 18m 386M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:37:30.070 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (5m) 10s ago 18m 342M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:37:30.071 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (4m) 10s ago 17m 355M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:37:30.071 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (3m) 10s ago 17m 388M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:37:30.071 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (3m) 12s ago 16m 340M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:37:30.071 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (2m) 12s ago 16m 304M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:37:30.071 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (114s) 12s ago 16m 254M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e4f91e53cf7d 2024-03-30T13:37:30.071 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (78s) 12s ago 15m 284M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 b2f37650410f 2024-03-30T13:37:30.071 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (8m) 10s ago 19m 69.8M - 2.33.4 514e6a882f6e 818f94aa47b2 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:37:30.531 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 8 2024-03-30T13:37:30.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:37:30.532 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:37:30.532 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:37:30.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:37:30.532 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:37:30.532 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 14 2024-03-30T13:37:30.532 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:37:30.532 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:37:30.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:30 smithi159 ceph-mon[67002]: from='client.34546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:37:30.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:30 smithi159 ceph-mon[67002]: from='client.34550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:37:30.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:30 smithi187 ceph-mon[53604]: from='client.34546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:37:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:30 smithi187 ceph-mon[53604]: from='client.34550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "nfs", 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "osd", 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "crash", 2024-03-30T13:37:30.887 INFO:teuthology.orchestra.run.smithi159.stdout: "mds", 2024-03-30T13:37:30.888 INFO:teuthology.orchestra.run.smithi159.stdout: "mon" 2024-03-30T13:37:30.888 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:37:30.888 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "18/23 daemons upgraded", 2024-03-30T13:37:30.888 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading node-exporter daemons", 2024-03-30T13:37:30.888 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:37:30.888 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:37:31.311 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:37:31.311 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:37:31.312 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:37:31.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:31 smithi159 ceph-mon[67002]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:37:31.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:31 smithi159 ceph-mon[67002]: pgmap v352: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 253 B/s rd, 0 op/s 2024-03-30T13:37:31.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:31 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/2912167620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:31.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:31 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/67862669' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:37:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:31 smithi187 ceph-mon[53604]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:37:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:31 smithi187 ceph-mon[53604]: pgmap v352: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 253 B/s rd, 0 op/s 2024-03-30T13:37:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:31 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/2912167620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:31.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:31 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/67862669' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:37:32.736 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:32 smithi159 ceph-mon[67002]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:37:32.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:32 smithi187 ceph-mon[53604]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:37:33.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:33 smithi159 ceph-mon[67002]: pgmap v353: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 253 B/s rd, 0 op/s 2024-03-30T13:37:33.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:33 smithi187 ceph-mon[53604]: pgmap v353: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 253 B/s rd, 0 op/s 2024-03-30T13:37:34.897 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:34.897 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:34 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:35.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:35.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:34 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:35.954 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:35 smithi187 ceph-mon[53604]: pgmap v354: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-30T13:37:36.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:35 smithi159 ceph-mon[67002]: pgmap v354: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-30T13:37:36.853 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:36 smithi187 ceph-mon[53604]: Upgrade: Updating node-exporter.smithi187 (2/2) 2024-03-30T13:37:36.853 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:36 smithi187 ceph-mon[53604]: Deploying daemon node-exporter.smithi187 on smithi187 2024-03-30T13:37:37.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:36 smithi159 ceph-mon[67002]: Upgrade: Updating node-exporter.smithi187 (2/2) 2024-03-30T13:37:37.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:36 smithi159 ceph-mon[67002]: Deploying daemon node-exporter.smithi187 on smithi187 2024-03-30T13:37:38.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:37 smithi159 ceph-mon[67002]: pgmap v355: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:38.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:37 smithi187 ceph-mon[53604]: pgmap v355: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:39.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:38 smithi159 ceph-mon[67002]: pgmap v356: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:39.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:38 smithi187 ceph-mon[53604]: pgmap v356: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:41.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:41 smithi159 ceph-mon[67002]: pgmap v357: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:37:41.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:41 smithi187 ceph-mon[53604]: pgmap v357: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:37:43.752 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:43 smithi159 ceph-mon[67002]: pgmap v358: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:43.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:43 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:37:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:43 smithi187 ceph-mon[53604]: pgmap v358: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:43.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:43 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:37:45.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:45 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:45.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:45 smithi159 ceph-mon[67002]: pgmap v359: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:45.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:45 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:45.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:45 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:45.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:45 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:45.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:45 smithi187 ceph-mon[53604]: pgmap v359: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:45.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:45 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:45.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:45 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:47.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:47 smithi159 ceph-mon[67002]: pgmap v360: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:47.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:47.751 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:47.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:47 smithi187 ceph-mon[53604]: pgmap v360: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:47.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:47.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:49.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:49 smithi187 ceph-mon[53604]: pgmap v361: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:49.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:49.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:49.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:49.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:49 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:49.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:49 smithi159 ceph-mon[67002]: pgmap v361: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:49.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:49.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:49.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:49.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:49 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:51.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:51 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:51.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:51 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:51.225 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:51 smithi159 ceph-mon[67002]: pgmap v362: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:37:51.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:51 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:51.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:51 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:51.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:51 smithi187 ceph-mon[53604]: pgmap v362: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:37:53.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:52 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:53.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:52 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:53.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:52 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:53.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:52 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:53.739 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:53 smithi159 ceph-mon[67002]: pgmap v363: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:54.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:53 smithi187 ceph-mon[53604]: pgmap v363: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:37:55.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:55.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:55.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:55.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:37:55.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: pgmap v364: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 172 B/s rd, 0 op/s 2024-03-30T13:37:55.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: pgmap v365: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:55 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: pgmap v364: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 172 B/s rd, 0 op/s 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: pgmap v365: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:55.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:55 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:37:57.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:57 smithi159 ceph-mon[67002]: Upgrade: Updating prometheus.smithi159 2024-03-30T13:37:57.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:57 smithi159 ceph-mon[67002]: pgmap v366: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:37:57.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:57 smithi159 ceph-mon[67002]: Deploying daemon prometheus.smithi159 on smithi159 2024-03-30T13:37:57.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:57 smithi187 ceph-mon[53604]: Upgrade: Updating prometheus.smithi159 2024-03-30T13:37:57.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:57 smithi187 ceph-mon[53604]: pgmap v366: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:37:57.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:57 smithi187 ceph-mon[53604]: Deploying daemon prometheus.smithi159 on smithi159 2024-03-30T13:37:58.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:37:58.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:37:59.415 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:37:59 smithi159 ceph-mon[67002]: pgmap v367: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:37:59.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:37:59 smithi187 ceph-mon[53604]: pgmap v367: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:38:01.280 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:01 smithi159 ceph-mon[67002]: pgmap v368: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:38:01.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:01 smithi187 ceph-mon[53604]: pgmap v368: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:38:01.726 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (11m) 13s ago 20m 25.4M - 0.23.0 ba2b418f427c f0e404f1a168 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (7m) 13s ago 20m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (7m) 15s ago 20m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 13s ago 20m - - 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (86s) 13s ago 14m 34.6M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 322b0f7af8dd 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (68s) 15s ago 14m 14.0M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 88d78b5e9b1d 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (10m) 13s ago 21m 513M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (9m) 15s ago 20m 453M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:38:02.514 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (9m) 13s ago 22m 61.7M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (8m) 15s ago 19m 48.1M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (53s) 13s ago 14m 57.7M - 5.7 b0c4478f1d83 785c7ee033e2 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (48s) 15s ago 14m 60.9M - 5.7 b0c4478f1d83 db1b13dcf23b 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (29s) 13s ago 20m 7260k - 1.5.0 0da6a335fe13 e4b12fe76715 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (18s) 15s ago 19m 5477k - 1.5.0 0da6a335fe13 62613faf3ba2 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (6m) 13s ago 19m 387M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (6m) 13s ago 18m 342M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (5m) 13s ago 18m 354M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (4m) 13s ago 17m 389M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (3m) 15s ago 17m 340M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (3m) 15s ago 17m 305M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (2m) 15s ago 16m 254M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e4f91e53cf7d 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (111s) 15s ago 16m 284M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 b2f37650410f 2024-03-30T13:38:02.515 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 starting - - - - 2024-03-30T13:38:02.943 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 8 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:38:02.944 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:38:02.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:38:02.945 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:38:02.945 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 14 2024-03-30T13:38:02.945 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:38:02.945 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:38:02.945 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:02.945 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:02.945 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:02.945 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:02.945 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:02 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:03.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:03.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:03.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:02 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:03.314 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:38:03.314 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:38:03.314 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "nfs", 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "osd", 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "crash", 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "mds", 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "mon" 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "18/23 daemons upgraded", 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading prometheus daemons", 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:38:03.315 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:38:03.756 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:38:03.756 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:38:03.756 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:38:04.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:03 smithi159 ceph-mon[67002]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:04.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:03 smithi159 ceph-mon[67002]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:04.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:03 smithi159 ceph-mon[67002]: pgmap v369: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:38:04.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:03 smithi159 ceph-mon[67002]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:04.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:03 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/2105332349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:04.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:03 smithi187 ceph-mon[53604]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:04.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:03 smithi187 ceph-mon[53604]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:04.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:03 smithi187 ceph-mon[53604]: pgmap v369: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-30T13:38:04.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:03 smithi187 ceph-mon[53604]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:04.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:03 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/2105332349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:05.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:04 smithi159 ceph-mon[67002]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:05.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:04 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1122107631' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:38:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:04 smithi187 ceph-mon[53604]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:04 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1122107631' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:38:06.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:05 smithi159 ceph-mon[67002]: pgmap v370: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-30T13:38:06.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:05 smithi187 ceph-mon[53604]: pgmap v370: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-30T13:38:08.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:07 smithi159 ceph-mon[67002]: pgmap v371: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:08.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:08.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:07 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:08.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:07 smithi187 ceph-mon[53604]: pgmap v371: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:08.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:08.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:07 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:09.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:09.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:09 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:09.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:09 smithi187 ceph-mon[53604]: pgmap v372: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:09.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:09.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:09 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:09.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:09 smithi159 ceph-mon[67002]: pgmap v372: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:11.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:11 smithi159 ceph-mon[67002]: pgmap v373: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:38:11.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:11 smithi187 ceph-mon[53604]: pgmap v373: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:38:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:12.863 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: pgmap v374: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 174 B/s rd, 0 op/s 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: pgmap v375: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.864 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:12.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: pgmap v374: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 174 B/s rd, 0 op/s 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: pgmap v375: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:13.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.273 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:13.274 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:38:14.214 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:13 smithi159 ceph-mon[67002]: Upgrade: Updating alertmanager.smithi159 2024-03-30T13:38:14.214 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:13 smithi159 ceph-mon[67002]: Deploying daemon alertmanager.smithi159 on smithi159 2024-03-30T13:38:14.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:13 smithi187 ceph-mon[53604]: Upgrade: Updating alertmanager.smithi159 2024-03-30T13:38:14.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:13 smithi187 ceph-mon[53604]: Deploying daemon alertmanager.smithi159 on smithi159 2024-03-30T13:38:15.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:14 smithi159 ceph-mon[67002]: pgmap v376: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:15.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:14 smithi187 ceph-mon[53604]: pgmap v376: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:17.197 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:16 smithi159 ceph-mon[67002]: pgmap v377: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:17.272 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:16 smithi187 ceph-mon[53604]: pgmap v377: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:19.136 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:18 smithi159 ceph-mon[67002]: pgmap v378: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:19.271 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:18 smithi187 ceph-mon[53604]: pgmap v378: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:20.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:20.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:20.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:20.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:20.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:20 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:20.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:20 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:21.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:21 smithi159 ceph-mon[67002]: pgmap v379: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:21.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:21 smithi187 ceph-mon[53604]: pgmap v379: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-03-30T13:38:23.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:23 smithi159 ceph-mon[67002]: pgmap v380: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-30T13:38:23.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:23 smithi187 ceph-mon[53604]: pgmap v380: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-30T13:38:25.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:25 smithi159 ceph-mon[67002]: pgmap v381: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:25.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:25.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:25 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:25 smithi187 ceph-mon[53604]: pgmap v381: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:25.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:25 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:26.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:26.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:26 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:26 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:26.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:26 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:27.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:27 smithi159 ceph-mon[67002]: pgmap v382: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:27.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:27 smithi187 ceph-mon[53604]: pgmap v382: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:28.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:28 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:38:28.771 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:28 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:38:29.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: pgmap v383: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:29.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:29.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:29.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.754 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.755 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:29 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: pgmap v383: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:29.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:29.773 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:29 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:30.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:30 smithi159 ceph-mon[67002]: pgmap v384: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 183 B/s rd, 0 op/s 2024-03-30T13:38:30.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:30 smithi159 ceph-mon[67002]: pgmap v385: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-30T13:38:30.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:30 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:38:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:30 smithi187 ceph-mon[53604]: pgmap v384: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 183 B/s rd, 0 op/s 2024-03-30T13:38:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:30 smithi187 ceph-mon[53604]: pgmap v385: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-30T13:38:30.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:30 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-30T13:38:32.680 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:32 smithi159 ceph-mon[67002]: pgmap v386: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-03-30T13:38:32.680 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:32 smithi159 ceph-mon[67002]: Upgrade: Updating grafana.smithi159 2024-03-30T13:38:32.680 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:32 smithi159 ceph-mon[67002]: Deploying daemon grafana.smithi159 on smithi159 2024-03-30T13:38:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:32 smithi187 ceph-mon[53604]: pgmap v386: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-03-30T13:38:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:32 smithi187 ceph-mon[53604]: Upgrade: Updating grafana.smithi159 2024-03-30T13:38:32.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:32 smithi187 ceph-mon[53604]: Deploying daemon grafana.smithi159 on smithi159 2024-03-30T13:38:34.137 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:38:34.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:34 smithi159 ceph-mon[67002]: pgmap v387: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-03-30T13:38:34.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:34 smithi187 ceph-mon[53604]: pgmap v387: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (16s) 10s ago 21m 16.2M - 0.25.0 c8568f914cd2 f5c0f7251d48 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (8m) 10s ago 21m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (7m) 47s ago 20m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 error 10s ago 20m - - 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (118s) 10s ago 15m 34.9M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 322b0f7af8dd 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (100s) 47s ago 15m 14.0M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 88d78b5e9b1d 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (10m) 10s ago 22m 515M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (9m) 47s ago 20m 453M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:38:34.881 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (9m) 10s ago 22m 62.6M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (9m) 47s ago 20m 48.1M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (86s) 10s ago 15m 58.0M - 5.7 b0c4478f1d83 785c7ee033e2 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (80s) 47s ago 15m 60.9M - 5.7 b0c4478f1d83 db1b13dcf23b 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (61s) 10s ago 20m 8120k - 1.5.0 0da6a335fe13 e4b12fe76715 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (51s) 47s ago 20m 5477k - 1.5.0 0da6a335fe13 62613faf3ba2 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (7m) 10s ago 19m 387M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (6m) 10s ago 19m 343M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (5m) 10s ago 18m 355M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (4m) 10s ago 18m 389M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (4m) 47s ago 18m 340M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (3m) 47s ago 17m 305M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (2m) 47s ago 17m 254M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e4f91e53cf7d 2024-03-30T13:38:34.882 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (2m) 47s ago 16m 284M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 b2f37650410f 2024-03-30T13:38:34.883 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (33s) 10s ago 20m 49.1M - 2.43.0 a07b618ecd1d 3be813624f49 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 8 2024-03-30T13:38:35.458 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:38:35.459 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:38:35.459 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:38:35.459 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:38:35.459 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:38:35.459 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 14 2024-03-30T13:38:35.459 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:38:35.459 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:38:35.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:35 smithi159 ceph-mon[67002]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:35.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:35 smithi159 ceph-mon[67002]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:35.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:35 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:35.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:35 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:35.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:35 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:35.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:35 smithi187 ceph-mon[53604]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:35.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:35 smithi187 ceph-mon[53604]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:35.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:35 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:35.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:35 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:35.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:35 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:35.940 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": true, 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [ 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "nfs", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "osd", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "crash", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "mds", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "mon" 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": "18/23 daemons upgraded", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "Currently upgrading grafana daemons", 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:38:35.941 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:38:36.458 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-30T13:38:36.458 INFO:teuthology.orchestra.run.smithi159.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-30T13:38:36.459 INFO:teuthology.orchestra.run.smithi159.stdout: daemon grafana.smithi159 on smithi159 is in error state 2024-03-30T13:38:36.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:36 smithi159 ceph-mon[67002]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:36.502 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:36 smithi159 ceph-mon[67002]: pgmap v388: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-03-30T13:38:36.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:36 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/2035950308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:36.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:36 smithi187 ceph-mon[53604]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:36.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:36 smithi187 ceph-mon[53604]: pgmap v388: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-03-30T13:38:36.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:36 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/2035950308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:37.673 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:37 smithi187 ceph-mon[53604]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:37.673 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:37 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1563140504' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:38:37.695 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:37 smithi159 ceph-mon[67002]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:38:37.695 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:37 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1563140504' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:38:38.633 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:38 smithi159 ceph-mon[67002]: pgmap v389: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-03-30T13:38:38.683 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:38 smithi187 ceph-mon[53604]: pgmap v389: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-03-30T13:38:40.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:40 smithi159 ceph-mon[67002]: pgmap v390: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-30T13:38:40.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:40 smithi187 ceph-mon[53604]: pgmap v390: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-30T13:38:42.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:41 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:42.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:41 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:42.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:41 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:42.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:41 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:43.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:42 smithi159 ceph-mon[67002]: pgmap v391: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:38:43.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:43.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:43.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:38:43.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:42 smithi187 ceph-mon[53604]: pgmap v391: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:38:43.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:43.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:43.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:38:44.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:44 smithi159 ceph-mon[67002]: pgmap v392: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:45.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:44 smithi187 ceph-mon[53604]: pgmap v392: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:46.503 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: pgmap v393: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: pgmap v394: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.504 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.505 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: Upgrade: Finalizing container_image settings 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-03-30T13:38:46.506 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.507 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: Upgrade: Complete! 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.508 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-30T13:38:46.509 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:46 smithi159 ceph-mon[67002]: Cluster is now healthy 2024-03-30T13:38:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: pgmap v393: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:46.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: pgmap v394: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.523 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: Upgrade: Finalizing container_image settings 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-03-30T13:38:46.524 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-03-30T13:38:46.525 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.526 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: Upgrade: Complete! 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-30T13:38:46.527 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:46 smithi187 ceph-mon[53604]: Cluster is now healthy 2024-03-30T13:38:49.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:48 smithi159 ceph-mon[67002]: pgmap v395: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:49.006 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:48 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:49.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:48 smithi187 ceph-mon[53604]: pgmap v395: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:49.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:48 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:51.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:50 smithi159 ceph-mon[67002]: pgmap v396: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:50 smithi187 ceph-mon[53604]: pgmap v396: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:53.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:52 smithi159 ceph-mon[67002]: pgmap v397: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:53.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:52 smithi187 ceph-mon[53604]: pgmap v397: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:55.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:54 smithi159 ceph-mon[67002]: pgmap v398: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:55.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:54 smithi187 ceph-mon[53604]: pgmap v398: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:57.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:56 smithi159 ceph-mon[67002]: pgmap v399: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:57.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:56 smithi187 ceph-mon[53604]: pgmap v399: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 200 B/s rd, 0 op/s 2024-03-30T13:38:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:58 smithi187 ceph-mon[53604]: pgmap v400: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:58.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:38:58 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:38:59.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:58 smithi159 ceph-mon[67002]: pgmap v400: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:38:59.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:38:59.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:38:58 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:00.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:00 smithi187 ceph-mon[53604]: pgmap v401: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:01.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:00 smithi159 ceph-mon[67002]: pgmap v401: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:02.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:02 smithi187 ceph-mon[53604]: pgmap v402: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:03.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:02 smithi159 ceph-mon[67002]: pgmap v402: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:04.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:04 smithi187 ceph-mon[53604]: pgmap v403: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:05.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:04 smithi159 ceph-mon[67002]: pgmap v403: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:06.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:06 smithi187 ceph-mon[53604]: pgmap v404: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:06.846 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:06 smithi159 ceph-mon[67002]: pgmap v404: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:07.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph orch ps' 2024-03-30T13:39:08.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:08 smithi187 ceph-mon[53604]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:39:08.772 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:08 smithi187 ceph-mon[53604]: pgmap v405: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:08.842 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:08 smithi159 ceph-mon[67002]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:39:08.843 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:08 smithi159 ceph-mon[67002]: pgmap v405: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:09.693 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:39:09.693 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (51s) 29s ago 21m 16.3M - 0.25.0 c8568f914cd2 f5c0f7251d48 2024-03-30T13:39:09.693 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (8m) 29s ago 21m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:39:09.693 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (8m) 82s ago 21m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:39:09.693 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (35s) 29s ago 21m 48.8M - 9.4.12 2bacad6d85d8 07f725bc2560 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (2m) 29s ago 16m 35.0M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 322b0f7af8dd 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (2m) 82s ago 15m 14.0M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 88d78b5e9b1d 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (11m) 29s ago 23m 522M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (10m) 82s ago 21m 453M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (10m) 29s ago 23m 63.5M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (9m) 82s ago 21m 48.1M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (2m) 29s ago 15m 58.2M - 5.7 b0c4478f1d83 785c7ee033e2 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (115s) 82s ago 15m 60.9M - 5.7 b0c4478f1d83 db1b13dcf23b 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (96s) 29s ago 21m 8154k - 1.5.0 0da6a335fe13 e4b12fe76715 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (85s) 82s ago 20m 5477k - 1.5.0 0da6a335fe13 62613faf3ba2 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (7m) 29s ago 20m 388M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (7m) 29s ago 19m 343M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (6m) 29s ago 19m 355M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (5m) 29s ago 18m 389M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:39:09.694 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (4m) 82s ago 18m 340M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:39:09.695 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (4m) 82s ago 18m 305M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:39:09.695 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (3m) 82s ago 17m 254M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e4f91e53cf7d 2024-03-30T13:39:09.695 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (2m) 82s ago 17m 284M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 b2f37650410f 2024-03-30T13:39:09.695 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (68s) 29s ago 20m 49.2M - 2.43.0 a07b618ecd1d 3be813624f49 2024-03-30T13:39:10.272 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph versions' 2024-03-30T13:39:10.821 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:10 smithi159 ceph-mon[67002]: pgmap v406: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:10.821 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:10 smithi159 ceph-mon[67002]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:39:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:10 smithi187 ceph-mon[53604]: pgmap v406: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:11.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:10 smithi187 ceph-mon[53604]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 8 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:39:12.297 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:39:12.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:39:12.298 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:39:12.298 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 14 2024-03-30T13:39:12.298 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:39:12.298 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:39:12.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:12 smithi159 ceph-mon[67002]: pgmap v407: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:12.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:12 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/346951598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:39:12.681 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:12.987 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-03-30T13:39:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:12 smithi187 ceph-mon[53604]: pgmap v407: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:12 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/346951598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:39:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:14.615 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:14 smithi159 ceph-mon[67002]: pgmap v408: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:14.841 INFO:teuthology.orchestra.run.smithi159.stdout:wait for servicemap items w/ changing names to refresh 2024-03-30T13:39:15.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:14 smithi187 ceph-mon[53604]: pgmap v408: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:15.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'sleep 60' 2024-03-30T13:39:16.802 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:16 smithi159 ceph-mon[67002]: pgmap v409: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:16 smithi187 ceph-mon[53604]: pgmap v409: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:19.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:18 smithi159 ceph-mon[67002]: pgmap v410: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:19.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:18 smithi187 ceph-mon[53604]: pgmap v410: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:21.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:20 smithi159 ceph-mon[67002]: pgmap v411: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:21.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:20 smithi187 ceph-mon[53604]: pgmap v411: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:23.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:22 smithi159 ceph-mon[67002]: pgmap v412: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:23.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:22 smithi187 ceph-mon[53604]: pgmap v412: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:25.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:24 smithi159 ceph-mon[67002]: pgmap v413: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:25.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:24 smithi187 ceph-mon[53604]: pgmap v413: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:27.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:26 smithi159 ceph-mon[67002]: pgmap v414: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:27.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:26 smithi187 ceph-mon[53604]: pgmap v414: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:28.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:27 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:28.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:27 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:29.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:28 smithi159 ceph-mon[67002]: pgmap v415: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:29.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:28 smithi187 ceph-mon[53604]: pgmap v415: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:30.804 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:30 smithi159 ceph-mon[67002]: pgmap v416: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:31.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:30 smithi187 ceph-mon[53604]: pgmap v416: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:33.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:32 smithi159 ceph-mon[67002]: pgmap v417: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:33.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:32 smithi187 ceph-mon[53604]: pgmap v417: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:35.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:34 smithi159 ceph-mon[67002]: pgmap v418: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:35.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:34 smithi187 ceph-mon[53604]: pgmap v418: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:37.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:36 smithi159 ceph-mon[67002]: pgmap v419: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:37.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:36 smithi187 ceph-mon[53604]: pgmap v419: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:38.824 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:38 smithi187 ceph-mon[53604]: pgmap v420: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:39.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:38 smithi159 ceph-mon[67002]: pgmap v420: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:41.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:40 smithi159 ceph-mon[67002]: pgmap v421: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:41.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:40 smithi187 ceph-mon[53604]: pgmap v421: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:43.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:42 smithi159 ceph-mon[67002]: pgmap v422: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:43.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:43.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:42 smithi187 ceph-mon[53604]: pgmap v422: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:43.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:45.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:44 smithi159 ceph-mon[67002]: pgmap v423: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:45.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:44 smithi187 ceph-mon[53604]: pgmap v423: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:46 smithi159 ceph-mon[67002]: pgmap v424: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:39:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:39:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:39:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:39:47.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:39:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:46 smithi187 ceph-mon[53604]: pgmap v424: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:39:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:39:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:39:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:39:47.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:39:49.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:48 smithi159 ceph-mon[67002]: pgmap v425: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:49.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:48 smithi187 ceph-mon[53604]: pgmap v425: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:51.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:50 smithi159 ceph-mon[67002]: pgmap v426: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:51.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:50 smithi187 ceph-mon[53604]: pgmap v426: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:53.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:52 smithi159 ceph-mon[67002]: pgmap v427: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:53.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:52 smithi187 ceph-mon[53604]: pgmap v427: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:39:55.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:54 smithi159 ceph-mon[67002]: pgmap v428: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:55.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:54 smithi187 ceph-mon[53604]: pgmap v428: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:57.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:56 smithi159 ceph-mon[67002]: pgmap v429: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:57.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:56 smithi187 ceph-mon[53604]: pgmap v429: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:58.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:57 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:58.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:57 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:39:59.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:39:58 smithi159 ceph-mon[67002]: pgmap v430: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:39:59.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:39:58 smithi187 ceph-mon[53604]: pgmap v430: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:01.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:00 smithi159 ceph-mon[67002]: pgmap v431: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:01.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:00 smithi159 ceph-mon[67002]: overall HEALTH_OK 2024-03-30T13:40:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:00 smithi187 ceph-mon[53604]: pgmap v431: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:01.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:00 smithi187 ceph-mon[53604]: overall HEALTH_OK 2024-03-30T13:40:03.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:02 smithi159 ceph-mon[67002]: pgmap v432: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:03.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:02 smithi187 ceph-mon[53604]: pgmap v432: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:05.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:04 smithi159 ceph-mon[67002]: pgmap v433: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:05.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:04 smithi187 ceph-mon[53604]: pgmap v433: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:07.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:06 smithi159 ceph-mon[67002]: pgmap v434: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:07.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:06 smithi187 ceph-mon[53604]: pgmap v434: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:09.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:08 smithi159 ceph-mon[67002]: pgmap v435: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:09.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:08 smithi187 ceph-mon[53604]: pgmap v435: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:11.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:10 smithi159 ceph-mon[67002]: pgmap v436: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:11.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:10 smithi187 ceph-mon[53604]: pgmap v436: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:13.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:12 smithi159 ceph-mon[67002]: pgmap v437: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:13.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:12 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:40:13.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:12 smithi187 ceph-mon[53604]: pgmap v437: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:13.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:12 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:40:15.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:14 smithi159 ceph-mon[67002]: pgmap v438: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:15.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:14 smithi187 ceph-mon[53604]: pgmap v438: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:17.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:16 smithi159 ceph-mon[67002]: pgmap v439: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:17.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:16 smithi187 ceph-mon[53604]: pgmap v439: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:17.830 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph orch ps' 2024-03-30T13:40:18.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:18 smithi159 ceph-mon[67002]: pgmap v440: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:19.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:18 smithi187 ceph-mon[53604]: pgmap v440: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:alertmanager.smithi159 smithi159 *:9093,9094 running (2m) 99s ago 23m 16.3M - 0.25.0 c8568f914cd2 f5c0f7251d48 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi159 smithi159 running (9m) 99s ago 23m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 e990fb3773bc 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:crash.smithi187 smithi187 running (9m) 2m ago 22m 6660k - 19.0.0-1967-g4e50bbcd b0c4478f1d83 daa5c5ba93a2 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:grafana.smithi159 smithi159 *:3000 running (105s) 99s ago 22m 48.8M - 9.4.12 2bacad6d85d8 07f725bc2560 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi159.jwwwqm smithi159 running (3m) 99s ago 17m 35.0M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 322b0f7af8dd 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:mds.foofs.smithi187.bkvdqd smithi187 running (3m) 2m ago 17m 14.0M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 88d78b5e9b1d 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi159.fnasly smithi159 *:8443,9283,8765 running (12m) 99s ago 24m 522M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 c09bd821f2fc 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:mgr.smithi187.jpkaud smithi187 *:8443,9283,8765 running (11m) 2m ago 22m 453M - 19.0.0-1967-g4e50bbcd b0c4478f1d83 86717b35c91d 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi159 smithi159 running (11m) 99s ago 24m 63.5M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 ba7c32cec301 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:mon.smithi187 smithi187 running (10m) 2m ago 22m 48.1M 2048M 19.0.0-1967-g4e50bbcd b0c4478f1d83 88b8025f24dd 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.0.0.smithi159.vhnwtw smithi159 *:2049 running (3m) 99s ago 16m 58.2M - 5.7 b0c4478f1d83 785c7ee033e2 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:nfs.foo.1.0.smithi187.dwmvfp smithi187 *:2049 running (3m) 2m ago 16m 60.9M - 5.7 b0c4478f1d83 db1b13dcf23b 2024-03-30T13:40:19.831 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi159 smithi159 *:9100 running (2m) 99s ago 22m 8154k - 1.5.0 0da6a335fe13 e4b12fe76715 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:node-exporter.smithi187 smithi187 *:9100 running (2m) 2m ago 22m 5477k - 1.5.0 0da6a335fe13 62613faf3ba2 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:osd.0 smithi159 running (9m) 99s ago 21m 388M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 a21add6ba181 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:osd.1 smithi159 running (8m) 99s ago 21m 343M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 2d27b1045a3d 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:osd.2 smithi159 running (7m) 99s ago 20m 355M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 56d8fa2c6a0f 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:osd.3 smithi159 running (6m) 99s ago 20m 389M 1936M 19.0.0-1967-g4e50bbcd b0c4478f1d83 addb9726df53 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:osd.4 smithi187 running (5m) 2m ago 19m 340M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 9611d3292acb 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:osd.5 smithi187 running (5m) 2m ago 19m 305M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e70ee05c8ec9 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:osd.6 smithi187 running (4m) 2m ago 19m 254M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 e4f91e53cf7d 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:osd.7 smithi187 running (4m) 2m ago 18m 284M 2704M 19.0.0-1967-g4e50bbcd b0c4478f1d83 b2f37650410f 2024-03-30T13:40:19.832 INFO:teuthology.orchestra.run.smithi159.stdout:prometheus.smithi159 smithi159 *:9095 running (2m) 99s ago 21m 49.2M - 2.43.0 a07b618ecd1d 3be813624f49 2024-03-30T13:40:20.477 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph orch upgrade status' 2024-03-30T13:40:20.742 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:20 smithi159 ceph-mon[67002]: pgmap v441: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:21.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:20 smithi187 ceph-mon[53604]: pgmap v441: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:21.946 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:21 smithi159 ceph-mon[67002]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:40:22.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:21 smithi187 ceph-mon[53604]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:40:22.523 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:40:22.523 INFO:teuthology.orchestra.run.smithi159.stdout: "target_image": null, 2024-03-30T13:40:22.523 INFO:teuthology.orchestra.run.smithi159.stdout: "in_progress": false, 2024-03-30T13:40:22.524 INFO:teuthology.orchestra.run.smithi159.stdout: "which": "", 2024-03-30T13:40:22.524 INFO:teuthology.orchestra.run.smithi159.stdout: "services_complete": [], 2024-03-30T13:40:22.524 INFO:teuthology.orchestra.run.smithi159.stdout: "progress": null, 2024-03-30T13:40:22.524 INFO:teuthology.orchestra.run.smithi159.stdout: "message": "", 2024-03-30T13:40:22.524 INFO:teuthology.orchestra.run.smithi159.stdout: "is_paused": false 2024-03-30T13:40:22.524 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:40:22.861 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:22 smithi159 ceph-mon[67002]: pgmap v442: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:23.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:22 smithi187 ceph-mon[53604]: pgmap v442: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:23.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph health detail' 2024-03-30T13:40:23.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:23 smithi159 ceph-mon[67002]: from='client.34634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:40:24.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:23 smithi187 ceph-mon[53604]: from='client.34634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:40:24.740 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:24 smithi159 ceph-mon[67002]: pgmap v443: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:25.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:24 smithi187 ceph-mon[53604]: pgmap v443: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:25.231 INFO:teuthology.orchestra.run.smithi159.stdout:HEALTH_OK 2024-03-30T13:40:25.850 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:25 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/2008411902' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:40:25.874 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph versions' 2024-03-30T13:40:26.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:25 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/2008411902' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-30T13:40:27.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:26 smithi159 ceph-mon[67002]: pgmap v444: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:27.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:26 smithi187 ceph-mon[53604]: pgmap v444: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:27.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:27 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:40:28.008 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-03-30T13:40:28.008 INFO:teuthology.orchestra.run.smithi159.stdout: "mon": { 2024-03-30T13:40:28.008 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:40:28.008 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:40:28.008 INFO:teuthology.orchestra.run.smithi159.stdout: "mgr": { 2024-03-30T13:40:28.008 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:40:28.008 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: "osd": { 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 8 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: "mds": { 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 2 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: "overall": { 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)": 14 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:40:28.009 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-03-30T13:40:28.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:27 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:40:28.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-03-30T13:40:28.958 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:28 smithi159 ceph-mon[67002]: pgmap v445: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:28.958 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:28 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/566602283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:40:29.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:28 smithi187 ceph-mon[53604]: pgmap v445: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:29.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:28 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/566602283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:40:30.715 INFO:teuthology.orchestra.run.smithi159.stdout:true 2024-03-30T13:40:31.002 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:30 smithi159 ceph-mon[67002]: pgmap v446: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:31.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:30 smithi187 ceph-mon[53604]: pgmap v446: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:31.313 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -e sha1=4e50bbcdd1630c7760b708f3a5b4d38a121e66cb -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-03-30T13:40:31.967 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:31 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/4152336318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:40:32.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:31 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/4152336318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:40:32.832 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:32 smithi159 ceph-mon[67002]: pgmap v447: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:33.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:32 smithi187 ceph-mon[53604]: pgmap v447: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:33.363 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph version 19.0.0-1967-g4e50bbcd (4e50bbcdd1630c7760b708f3a5b4d38a121e66cb) squid (dev)" 2024-03-30T13:40:33.978 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:33 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1602635901' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:40:34.002 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-03-30T13:40:34.003 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-03-30T13:40:34.003 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- ceph orch ls -f json 2024-03-30T13:40:34.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:33 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1602635901' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-30T13:40:35.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:34 smithi159 ceph-mon[67002]: pgmap v448: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:35.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:34 smithi187 ceph-mon[53604]: pgmap v448: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:36.036 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-03-30T13:40:36.036 INFO:teuthology.orchestra.run.smithi159.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-30T13:16:39.794254Z", "last_refresh": "2024-03-30T13:38:40.546083Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-30T13:16:34.339500Z", "last_refresh": "2024-03-30T13:37:47.246424Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-03-30T13:16:37.087366Z", "last_refresh": "2024-03-30T13:38:40.546250Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-30T13:23:04.210243Z", "last_refresh": "2024-03-30T13:37:47.247102Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-30T13:16:33.126309Z", "last_refresh": "2024-03-30T13:37:47.246527Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi159:172.21.15.159=smithi159", "smithi187:172.21.15.187=smithi187"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-30T13:17:21.867431Z", "last_refresh": "2024-03-30T13:37:47.246620Z", "running": 2, "size": 2}}, {"events": ["2024-03-30T13:39:46.319403Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-03-30T13:23:22.223601Z", "last_refresh": "2024-03-30T13:37:47.247176Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-30T13:16:38.180123Z", "last_refresh": "2024-03-30T13:37:47.246693Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d", "last_refresh": "2024-03-30T13:37:47.246765Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-30T13:16:35.797099Z", "last_refresh": "2024-03-30T13:38:40.546398Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-30T13:40:36.668 INFO:tasks.cephadm:nfs.foo has 2/2 2024-03-30T13:40:36.668 DEBUG:teuthology.parallel:result is None 2024-03-30T13:40:36.668 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-30T13:40:36.681 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:40:36.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-03-30T13:40:36.709 INFO:teuthology.orchestra.run.smithi159.stderr:+ umount /mnt/foo 2024-03-30T13:40:36.721 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-30T13:40:36.730 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:40:36.730 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-03-30T13:40:37.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:36 smithi159 ceph-mon[67002]: pgmap v449: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:37.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:36 smithi187 ceph-mon[53604]: pgmap v449: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:38.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:37 smithi159 ceph-mon[67002]: from='client.34654 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:40:38.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:37 smithi187 ceph-mon[53604]: from='client.34654 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-30T13:40:38.921 INFO:teuthology.orchestra.run.smithi159.stdout: "foo" 2024-03-30T13:40:38.921 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:38 smithi159 ceph-mon[67002]: pgmap v450: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:38.923 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:38 smithi187 ceph-mon[53604]: pgmap v450: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:39.582 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'ceph nfs export ls foo --detailed' 2024-03-30T13:40:41.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:40 smithi159 ceph-mon[67002]: from='client.44467 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:40:41.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:40 smithi159 ceph-mon[67002]: pgmap v451: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:41.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:40 smithi187 ceph-mon[53604]: from='client.44467 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:40:41.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:40 smithi187 ceph-mon[53604]: pgmap v451: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-30T13:40:41.875 INFO:teuthology.orchestra.run.smithi159.stdout:[ 2024-03-30T13:40:41.875 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "access_type": "RW", 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "clients": [], 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "cluster_id": "foo", 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "export_id": 1, 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "fsal": { 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "fs_name": "foofs", 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "name": "CEPH", 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "user_id": "nfs.foo.1" 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "path": "/", 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "protocols": [ 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: 4 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-03-30T13:40:41.876 INFO:teuthology.orchestra.run.smithi159.stdout: "pseudo": "/fake", 2024-03-30T13:40:41.877 INFO:teuthology.orchestra.run.smithi159.stdout: "security_label": true, 2024-03-30T13:40:41.877 INFO:teuthology.orchestra.run.smithi159.stdout: "squash": "none", 2024-03-30T13:40:41.877 INFO:teuthology.orchestra.run.smithi159.stdout: "transports": [ 2024-03-30T13:40:41.877 INFO:teuthology.orchestra.run.smithi159.stdout: "TCP" 2024-03-30T13:40:41.877 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-03-30T13:40:41.877 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-03-30T13:40:41.877 INFO:teuthology.orchestra.run.smithi159.stdout:] 2024-03-30T13:40:42.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'rados -p .nfs --all ls -' 2024-03-30T13:40:42.865 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:42 smithi159 ceph-mon[67002]: pgmap v452: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:42.866 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:42 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:40:43.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:42 smithi187 ceph-mon[53604]: pgmap v452: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-30T13:40:43.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:42 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-30T13:40:44.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:43 smithi159 ceph-mon[67002]: from='client.34662 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:40:44.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:43 smithi159 ceph-mon[67002]: mgrmap e44: smithi159.fnasly(active, since 11m), standbys: smithi187.jpkaud 2024-03-30T13:40:44.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:43 smithi187 ceph-mon[53604]: from='client.34662 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-30T13:40:44.022 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:43 smithi187 ceph-mon[53604]: mgrmap e44: smithi159.fnasly(active, since 11m), standbys: smithi187.jpkaud 2024-03-30T13:40:44.402 INFO:teuthology.orchestra.run.smithi159.stdout:foo rec-0000000000000004:nfs.foo.1 2024-03-30T13:40:44.402 INFO:teuthology.orchestra.run.smithi159.stdout:foo grace 2024-03-30T13:40:44.402 INFO:teuthology.orchestra.run.smithi159.stdout:foo rec-0000000000000004:nfs.foo.0 2024-03-30T13:40:44.402 INFO:teuthology.orchestra.run.smithi159.stdout:foo conf-nfs.foo 2024-03-30T13:40:44.402 INFO:teuthology.orchestra.run.smithi159.stdout:foo export-1 2024-03-30T13:40:45.003 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:44 smithi159 ceph-mon[67002]: pgmap v453: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-30T13:40:45.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:44 smithi187 ceph-mon[53604]: pgmap v453: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-30T13:40:45.112 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-30T13:40:45.123 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:40:45.123 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 -- bash -c 'set -ex 2024-03-30T13:40:45.123 DEBUG:teuthology.orchestra.run.smithi159:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-03-30T13:40:45.123 DEBUG:teuthology.orchestra.run.smithi159:> ' 2024-03-30T13:40:46.728 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:46 smithi159 ceph-mon[67002]: pgmap v454: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-30T13:40:46.729 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:46 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:40:46.912 INFO:teuthology.orchestra.run.smithi159.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-03-30T13:40:47.021 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:46 smithi187 ceph-mon[53604]: pgmap v454: 97 pgs: 97 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-30T13:40:47.077 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:46 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-30T13:40:47.236 INFO:teuthology.orchestra.run.smithi159.stderr:+ [[ 6 -gt 2 ]] 2024-03-30T13:40:47.837 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:40:47.837 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:40:47.837 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:40:47.837 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:47 smithi159 ceph-mon[67002]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:40:47.837 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:47 smithi159 ceph-mon[67002]: from='client.? 172.21.15.159:0/1787987779' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-03-30T13:40:47.864 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-30T13:40:47.875 INFO:tasks.cephadm:Teardown begin 2024-03-30T13:40:47.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-30T13:40:47.905 DEBUG:teuthology.orchestra.run.smithi187:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-30T13:40:47.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-30T13:40:47.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-30T13:40:47.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:40:47.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:47 smithi187 ceph-mon[53604]: from='mgr.34104 172.21.15.159:0/3661766228' entity='mgr.smithi159.fnasly' 2024-03-30T13:40:47.937 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:47 smithi187 ceph-mon[53604]: from='client.? 172.21.15.159:0/1787987779' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-03-30T13:40:47.940 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-30T13:40:47.940 DEBUG:teuthology.orchestra.run.smithi159:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-30T13:40:47.959 DEBUG:teuthology.orchestra.run.smithi187:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-30T13:40:47.996 INFO:tasks.cephadm:Stopping all daemons... 2024-03-30T13:40:47.996 INFO:tasks.cephadm.mon.smithi159:Stopping mon.smithi159... 2024-03-30T13:40:47.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159 2024-03-30T13:40:48.253 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:48 smithi159 systemd[1]: Stopping Ceph mon.smithi159 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:40:48.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[66998]: 2024-03-30T13:40:48.329+0000 7ff77109c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi159 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:40:48.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:48 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159[66998]: 2024-03-30T13:40:48.329+0000 7ff77109c700 -1 mon.smithi159@0(leader) e3 *** Got Signal Terminated *** 2024-03-30T13:40:48.753 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:48 smithi159 podman[112878]: 2024-03-30 13:40:48.554489884 +0000 UTC m=+0.341445996 container died ba7c32cec3016cd068a8a55c27b9a0b5afa45402281a06e6f30377e22501b23d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:40:49.191 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:48 smithi159 podman[112878]: 2024-03-30 13:40:48.897858019 +0000 UTC m=+0.684814138 container cleanup ba7c32cec3016cd068a8a55c27b9a0b5afa45402281a06e6f30377e22501b23d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:40:49.191 INFO:journalctl@ceph.mon.smithi159.smithi159.stdout:Mar 30 13:40:48 smithi159 bash[112878]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi159 2024-03-30T13:40:49.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi159.service' 2024-03-30T13:40:49.407 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:40:49.407 INFO:tasks.cephadm.mon.smithi159:Stopped mon.smithi159 2024-03-30T13:40:49.407 INFO:tasks.cephadm.mon.smithi187:Stopping mon.smithi187... 2024-03-30T13:40:49.408 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi187 2024-03-30T13:40:49.708 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:49 smithi187 systemd[1]: Stopping Ceph mon.smithi187 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:40:49.963 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:49 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187[53600]: 2024-03-30T13:40:49.706+0000 7fe439aa6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi187 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:40:49.963 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:49 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187[53600]: 2024-03-30T13:40:49.706+0000 7fe439aa6700 -1 mon.smithi187@1(peon) e3 *** Got Signal Terminated *** 2024-03-30T13:40:50.259 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:49 smithi187 podman[79925]: 2024-03-30 13:40:49.962165154 +0000 UTC m=+0.356724165 container died 88b8025f24dd2ea3b4d42ad60f11bc72962f4344245fde97ce18bd23d8a8470a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-30T13:40:50.521 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:50 smithi187 podman[79925]: 2024-03-30 13:40:50.457636768 +0000 UTC m=+0.852195778 container cleanup 88b8025f24dd2ea3b4d42ad60f11bc72962f4344245fde97ce18bd23d8a8470a (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:40:50.522 INFO:journalctl@ceph.mon.smithi187.smithi187.stdout:Mar 30 13:40:50 smithi187 bash[79925]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-mon-smithi187 2024-03-30T13:40:50.874 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@mon.smithi187.service' 2024-03-30T13:40:50.923 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:40:50.924 INFO:tasks.cephadm.mon.smithi187:Stopped mon.smithi187 2024-03-30T13:40:50.924 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-30T13:40:50.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.0 2024-03-30T13:40:51.225 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:50 smithi159 systemd[1]: Stopping Ceph osd.0 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:40:51.503 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:51 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[73471]: 2024-03-30T13:40:51.242+0000 7f6baea16700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:40:51.503 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:51 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[73471]: 2024-03-30T13:40:51.242+0000 7f6baea16700 -1 osd.0 155 *** Got signal Terminated *** 2024-03-30T13:40:51.503 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:51 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0[73471]: 2024-03-30T13:40:51.242+0000 7f6baea16700 -1 osd.0 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:40:57.113 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:56 smithi159 podman[113043]: 2024-03-30 13:40:56.795125165 +0000 UTC m=+5.654346214 container died a21add6ba181b43ae9d607fc3b0216eb387dbf6eb5e24fa8b72e3b016d31c9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-30T13:40:57.454 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:57 smithi159 podman[113043]: 2024-03-30 13:40:57.194190278 +0000 UTC m=+6.053411312 container cleanup a21add6ba181b43ae9d607fc3b0216eb387dbf6eb5e24fa8b72e3b016d31c9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-30T13:40:57.455 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:57 smithi159 bash[113043]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0 2024-03-30T13:40:57.753 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:57 smithi159 podman[113061]: 2024-03-30 13:40:57.454143947 +0000 UTC m=+0.661785264 container remove a21add6ba181b43ae9d607fc3b0216eb387dbf6eb5e24fa8b72e3b016d31c9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-03-30T13:40:58.142 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:57 smithi159 podman[113153]: 2024-03-30 13:40:57.723604817 +0000 UTC m=+0.019647558 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:40:58.142 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:57 smithi159 podman[113153]: 2024-03-30 13:40:57.846858881 +0000 UTC m=+0.142901584 container create 3b03dc046b5972e9e272956388b6c43dda6f6e206dfef4c9c22c94b40b4fb92f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-03-30T13:40:58.142 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:58 smithi159 podman[113153]: 2024-03-30 13:40:58.057716265 +0000 UTC m=+0.353758970 container init 3b03dc046b5972e9e272956388b6c43dda6f6e206dfef4c9c22c94b40b4fb92f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux ) 2024-03-30T13:40:58.142 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:58 smithi159 podman[113153]: 2024-03-30 13:40:58.061090855 +0000 UTC m=+0.357133565 container start 3b03dc046b5972e9e272956388b6c43dda6f6e206dfef4c9c22c94b40b4fb92f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-03-30T13:40:58.142 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:58 smithi159 podman[113153]: 2024-03-30 13:40:58.141145369 +0000 UTC m=+0.437188085 container attach 3b03dc046b5972e9e272956388b6c43dda6f6e206dfef4c9c22c94b40b4fb92f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-30T13:40:58.503 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 30 13:40:58 smithi159 podman[113153]: 2024-03-30 13:40:58.366596898 +0000 UTC m=+0.662639609 container died 3b03dc046b5972e9e272956388b6c43dda6f6e206dfef4c9c22c94b40b4fb92f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, io.buildah.version=1.33.5) 2024-03-30T13:40:58.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.0.service' 2024-03-30T13:40:59.032 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:40:59.033 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-30T13:40:59.033 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-30T13:40:59.033 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.1 2024-03-30T13:40:59.253 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:40:59 smithi159 systemd[1]: Stopping Ceph osd.1 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:40:59.753 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:40:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:40:59.447+0000 7f187b36f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:40:59.753 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:40:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:40:59.447+0000 7f187b36f700 -1 osd.1 155 *** Got signal Terminated *** 2024-03-30T13:40:59.753 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:40:59 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1[76105]: 2024-03-30T13:40:59.447+0000 7f187b36f700 -1 osd.1 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:41:05.141 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:04 smithi159 podman[113257]: 2024-03-30 13:41:04.850526903 +0000 UTC m=+5.538969062 container died 2d27b1045a3df567528c4fc35314db0e9df9357dc4918f34e1367fcb03aa334d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:41:05.492 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:05 smithi159 podman[113257]: 2024-03-30 13:41:05.214610315 +0000 UTC m=+5.903052488 container cleanup 2d27b1045a3df567528c4fc35314db0e9df9357dc4918f34e1367fcb03aa334d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326) 2024-03-30T13:41:05.492 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:05 smithi159 bash[113257]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1 2024-03-30T13:41:05.753 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:05 smithi159 podman[113286]: 2024-03-30 13:41:05.491453365 +0000 UTC m=+0.642992833 container remove 2d27b1045a3df567528c4fc35314db0e9df9357dc4918f34e1367fcb03aa334d (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-30T13:41:06.177 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:05 smithi159 podman[113376]: 2024-03-30 13:41:05.756574275 +0000 UTC m=+0.024501088 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:41:06.177 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:05 smithi159 podman[113376]: 2024-03-30 13:41:05.867349445 +0000 UTC m=+0.135276241 container create 953870bf693d6e1032deae65ef3737ff2e0a2c1c6b13a9096eacc3afa45adef9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2) 2024-03-30T13:41:06.177 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:06 smithi159 podman[113376]: 2024-03-30 13:41:06.092587558 +0000 UTC m=+0.360514375 container init 953870bf693d6e1032deae65ef3737ff2e0a2c1c6b13a9096eacc3afa45adef9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-30T13:41:06.177 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:06 smithi159 podman[113376]: 2024-03-30 13:41:06.095771475 +0000 UTC m=+0.363698273 container start 953870bf693d6e1032deae65ef3737ff2e0a2c1c6b13a9096eacc3afa45adef9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, RELEASE=squid-4e50bbc, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-30T13:41:06.177 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:06 smithi159 podman[113376]: 2024-03-30 13:41:06.175945731 +0000 UTC m=+0.443872527 container attach 953870bf693d6e1032deae65ef3737ff2e0a2c1c6b13a9096eacc3afa45adef9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS) 2024-03-30T13:41:06.746 INFO:journalctl@ceph.osd.1.smithi159.stdout:Mar 30 13:41:06 smithi159 podman[113376]: 2024-03-30 13:41:06.439638275 +0000 UTC m=+0.707565084 container died 953870bf693d6e1032deae65ef3737ff2e0a2c1c6b13a9096eacc3afa45adef9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-03-30T13:41:07.050 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.1.service' 2024-03-30T13:41:07.093 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:41:07.094 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-30T13:41:07.094 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-30T13:41:07.094 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.2 2024-03-30T13:41:07.433 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:07 smithi159 systemd[1]: Stopping Ceph osd.2 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:41:07.753 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:07 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[78718]: 2024-03-30T13:41:07.432+0000 7fe69da40700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:41:07.753 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:07 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[78718]: 2024-03-30T13:41:07.432+0000 7fe69da40700 -1 osd.2 155 *** Got signal Terminated *** 2024-03-30T13:41:07.753 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:07 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2[78718]: 2024-03-30T13:41:07.432+0000 7fe69da40700 -1 osd.2 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:41:13.085 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:12 smithi159 podman[113481]: 2024-03-30 13:41:12.826497495 +0000 UTC m=+5.501893026 container died 56d8fa2c6a0f7ad9d92ea3d48a28f7550b2edc7d8074f8fa5a798c10456f4ec8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:41:13.451 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:13 smithi159 podman[113481]: 2024-03-30 13:41:13.182512745 +0000 UTC m=+5.857908274 container cleanup 56d8fa2c6a0f7ad9d92ea3d48a28f7550b2edc7d8074f8fa5a798c10456f4ec8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-30T13:41:13.451 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:13 smithi159 bash[113481]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2 2024-03-30T13:41:13.753 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:13 smithi159 podman[113535]: 2024-03-30 13:41:13.450426329 +0000 UTC m=+0.619306827 container remove 56d8fa2c6a0f7ad9d92ea3d48a28f7550b2edc7d8074f8fa5a798c10456f4ec8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-30T13:41:14.121 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:13 smithi159 podman[113625]: 2024-03-30 13:41:13.717730569 +0000 UTC m=+0.019040412 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:41:14.121 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:13 smithi159 podman[113625]: 2024-03-30 13:41:13.818048545 +0000 UTC m=+0.119358389 container create b8ba7b7f3e8211243d5ffad0f0d2293e41ff05b489a2388eff7816edf57173f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-03-30T13:41:14.121 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:14 smithi159 podman[113625]: 2024-03-30 13:41:14.037185707 +0000 UTC m=+0.338495552 container init b8ba7b7f3e8211243d5ffad0f0d2293e41ff05b489a2388eff7816edf57173f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326) 2024-03-30T13:41:14.121 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:14 smithi159 podman[113625]: 2024-03-30 13:41:14.040296325 +0000 UTC m=+0.341606176 container start b8ba7b7f3e8211243d5ffad0f0d2293e41ff05b489a2388eff7816edf57173f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, org.label-schema.build-date=20240326, ceph=True) 2024-03-30T13:41:14.121 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:14 smithi159 podman[113625]: 2024-03-30 13:41:14.120521248 +0000 UTC m=+0.421831094 container attach b8ba7b7f3e8211243d5ffad0f0d2293e41ff05b489a2388eff7816edf57173f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc) 2024-03-30T13:41:14.503 INFO:journalctl@ceph.osd.2.smithi159.stdout:Mar 30 13:41:14 smithi159 podman[113625]: 2024-03-30 13:41:14.351277208 +0000 UTC m=+0.652587046 container died b8ba7b7f3e8211243d5ffad0f0d2293e41ff05b489a2388eff7816edf57173f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:41:14.986 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.2.service' 2024-03-30T13:41:15.026 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:41:15.026 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-30T13:41:15.026 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-30T13:41:15.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.3 2024-03-30T13:41:15.253 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:15 smithi159 systemd[1]: Stopping Ceph osd.3 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:41:15.753 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:15 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[81274]: 2024-03-30T13:41:15.360+0000 7f700afeb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:41:15.753 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:15 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[81274]: 2024-03-30T13:41:15.360+0000 7f700afeb700 -1 osd.3 155 *** Got signal Terminated *** 2024-03-30T13:41:15.753 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:15 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[81274]: 2024-03-30T13:41:15.360+0000 7f700afeb700 -1 osd.3 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:41:18.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:17 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:17.521+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:18.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:18 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:18.512+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:18.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:18 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:18.384+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:19.471 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:19.352+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:19.771 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:19 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:19.470+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:20.002 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:19 smithi159 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3[81274]: 2024-03-30T13:41:19.746+0000 7f70035d7700 -1 osd.3 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.517304+0000 front 2024-03-30T13:40:55.517217+0000 (oldest deadline 2024-03-30T13:41:18.917624+0000) 2024-03-30T13:41:20.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:20 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:20.455+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:20.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:20 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:20.359+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:21.224 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:20 smithi159 podman[113730]: 2024-03-30 13:41:20.958938986 +0000 UTC m=+5.705230949 container died addb9726df53392c51b550704d139eb981727fd696ae1029b9fc4d3b8018d64b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-03-30T13:41:21.503 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:21 smithi159 podman[113730]: 2024-03-30 13:41:21.410406765 +0000 UTC m=+6.156698718 container cleanup addb9726df53392c51b550704d139eb981727fd696ae1029b9fc4d3b8018d64b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:41:21.503 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:21 smithi159 bash[113730]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3 2024-03-30T13:41:21.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:21 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:21.419+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:21.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:21 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:21.332+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:21.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:21 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:21.446+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:22.003 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:21 smithi159 podman[113750]: 2024-03-30 13:41:21.695203791 +0000 UTC m=+0.734605906 container remove addb9726df53392c51b550704d139eb981727fd696ae1029b9fc4d3b8018d64b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-30T13:41:22.333 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:22 smithi159 podman[113839]: 2024-03-30 13:41:21.969682271 +0000 UTC m=+0.018996917 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:41:22.333 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:22 smithi159 podman[113839]: 2024-03-30 13:41:22.096134292 +0000 UTC m=+0.145448931 container create e25c18b81ed7d891914154876303deb0b0599021ce77d886083354f0fad21164 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, RELEASE=squid-4e50bbc, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-30T13:41:22.333 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:22 smithi159 podman[113839]: 2024-03-30 13:41:22.332195993 +0000 UTC m=+0.381510629 container init e25c18b81ed7d891914154876303deb0b0599021ce77d886083354f0fad21164 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc) 2024-03-30T13:41:22.643 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:22 smithi159 podman[113839]: 2024-03-30 13:41:22.335473995 +0000 UTC m=+0.384788638 container start e25c18b81ed7d891914154876303deb0b0599021ce77d886083354f0fad21164 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-30T13:41:22.643 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:22 smithi159 podman[113839]: 2024-03-30 13:41:22.432360709 +0000 UTC m=+0.481675354 container attach e25c18b81ed7d891914154876303deb0b0599021ce77d886083354f0fad21164 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc) 2024-03-30T13:41:22.771 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:22 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:22.428+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:22.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:22 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:22.583+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:22.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:22 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:22.402+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:22.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:22 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:22.353+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:22.953 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 30 13:41:22 smithi159 podman[113839]: 2024-03-30 13:41:22.643073656 +0000 UTC m=+0.692388306 container died e25c18b81ed7d891914154876303deb0b0599021ce77d886083354f0fad21164 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5) 2024-03-30T13:41:23.306 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.3.service' 2024-03-30T13:41:23.346 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:41:23.346 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-30T13:41:23.346 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-30T13:41:23.346 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.4 2024-03-30T13:41:23.609 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:23.414+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:23.610 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:23.323+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:23.610 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:23 smithi187 systemd[1]: Stopping Ceph osd.4 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:41:23.610 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:23.465+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:24.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:23.605+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:24.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:23.649+0000 7fde6ec53700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:41:24.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:23.649+0000 7fde6ec53700 -1 osd.4 155 *** Got signal Terminated *** 2024-03-30T13:41:24.022 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:23 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:23.649+0000 7fde6ec53700 -1 osd.4 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:41:24.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:24 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:24.464+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:24.773 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:24 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:24.568+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:24.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:24 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:24.402+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:24.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:24 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:24.373+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:25.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:25 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:25.482+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:25.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:25 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:25.370+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:25.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:25 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:25.606+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:25.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:25 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:25.429+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:26.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:26 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:26.591+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:26.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:26 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:26.591+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:26.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:26 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:26.391+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:26.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:26 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:26.401+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:26.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:26 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:26.401+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:26.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:26 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:26.483+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:26.773 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:26 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:26.483+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:02.062848+0000 front 2024-03-30T13:41:02.063080+0000 (oldest deadline 2024-03-30T13:41:25.562994+0000) 2024-03-30T13:41:27.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:27 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:27.514+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:27.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:27 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:27.514+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:02.062848+0000 front 2024-03-30T13:41:02.063080+0000 (oldest deadline 2024-03-30T13:41:25.562994+0000) 2024-03-30T13:41:27.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:27 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:27.375+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:27.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:27 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:27.375+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:27.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:27 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:27.441+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:27.773 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:27 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:27.587+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:27.773 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:27 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:27.587+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:28.683 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:28.508+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.861059+0000 front 2024-03-30T13:40:55.861104+0000 (oldest deadline 2024-03-30T13:41:16.961033+0000) 2024-03-30T13:41:28.683 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4[58526]: 2024-03-30T13:41:28.508+0000 7fde6723f700 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:02.062848+0000 front 2024-03-30T13:41:02.063080+0000 (oldest deadline 2024-03-30T13:41:25.562994+0000) 2024-03-30T13:41:28.683 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:28.581+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:28.684 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:28.581+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:28.684 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:28.439+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:28.684 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:28.439+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:28.684 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:28.418+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:28.684 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:28 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:28.418+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:29.385 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:29 smithi187 podman[80123]: 2024-03-30 13:41:29.129681469 +0000 UTC m=+5.584857986 container died 9611d3292acb167c37b5ac7f4d00467c9dc2560d899414d2b5c66720e1756739 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-30T13:41:29.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:29 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:29.431+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:29.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:29 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:29.431+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:29.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:29 smithi187 bash[80123]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4 2024-03-30T13:41:29.772 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:29 smithi187 podman[80123]: 2024-03-30 13:41:29.575277542 +0000 UTC m=+6.030454076 container cleanup 9611d3292acb167c37b5ac7f4d00467c9dc2560d899414d2b5c66720e1756739 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-30T13:41:29.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:29 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:29.573+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:29.773 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:29 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:29.573+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:29.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:29 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:29.428+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:29.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:29 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:29.428+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:30.189 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:29 smithi187 podman[80142]: 2024-03-30 13:41:29.860345557 +0000 UTC m=+0.729792018 container remove 9611d3292acb167c37b5ac7f4d00467c9dc2560d899414d2b5c66720e1756739 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-03-30T13:41:30.458 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:30 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:30.433+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:30.459 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:30 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:30.433+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:30.459 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:30 smithi187 podman[80265]: 2024-03-30 13:41:30.091473078 +0000 UTC m=+0.017662601 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:41:30.459 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:30 smithi187 podman[80265]: 2024-03-30 13:41:30.194904117 +0000 UTC m=+0.121093668 container create 70d1d080dccc0064cd3b9c1050b110e8c5a6849c4ffe775f200c32f7b3c8e60b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-30T13:41:30.459 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:30 smithi187 podman[80265]: 2024-03-30 13:41:30.431137664 +0000 UTC m=+0.357327187 container init 70d1d080dccc0064cd3b9c1050b110e8c5a6849c4ffe775f200c32f7b3c8e60b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-30T13:41:30.459 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:30 smithi187 podman[80265]: 2024-03-30 13:41:30.43425108 +0000 UTC m=+0.360440591 container start 70d1d080dccc0064cd3b9c1050b110e8c5a6849c4ffe775f200c32f7b3c8e60b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-4e50bbc, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-30T13:41:30.730 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:30 smithi187 podman[80265]: 2024-03-30 13:41:30.547875155 +0000 UTC m=+0.474064689 container attach 70d1d080dccc0064cd3b9c1050b110e8c5a6849c4ffe775f200c32f7b3c8e60b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-4e50bbc) 2024-03-30T13:41:30.731 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:30 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:30.456+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:30.731 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:30 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:30.456+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:30.731 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:30 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:30.540+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:30.731 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:30 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:30.540+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:31.010 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 30 13:41:30 smithi187 podman[80265]: 2024-03-30 13:41:30.730528038 +0000 UTC m=+0.656717554 container died 70d1d080dccc0064cd3b9c1050b110e8c5a6849c4ffe775f200c32f7b3c8e60b (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-4-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:41:31.313 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.4.service' 2024-03-30T13:41:31.354 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:41:31.354 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-30T13:41:31.354 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-30T13:41:31.355 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.5 2024-03-30T13:41:31.551 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:31.460+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:31.551 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:31.460+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:31.552 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:31.465+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:31.552 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:31.465+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:31.552 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:31 smithi187 systemd[1]: Stopping Ceph osd.5 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:41:31.552 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:31.528+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:31.553 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:31.528+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:32.021 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:31.703+0000 7f885ab82700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:41:32.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:31.703+0000 7f885ab82700 -1 osd.5 155 *** Got signal Terminated *** 2024-03-30T13:41:32.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:31 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:31.703+0000 7f885ab82700 -1 osd.5 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:41:32.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:32 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:32.559+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:32.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:32 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:32.559+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:32.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:32 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:32.501+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:32.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:32 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:32.501+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:32.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:32 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:32.439+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:32.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:32 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:32.439+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:33.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:33 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:33.600+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:33.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:33 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:33.600+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:33.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:33 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:33.477+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:33.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:33 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:33.477+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:33.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:33 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:33.485+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:33.773 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:33 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:33.485+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:34.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:34.623+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:34.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:34.623+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:34.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:34.623+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:09.171626+0000 front 2024-03-30T13:41:09.171583+0000 (oldest deadline 2024-03-30T13:41:33.871666+0000) 2024-03-30T13:41:34.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:34.528+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:34.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:34.528+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:34.447+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:34.447+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:34.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:34 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:34.447+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:35.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:35 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:35.662+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:35.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:35 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:35.662+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:35.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:35 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:35.662+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:09.171626+0000 front 2024-03-30T13:41:09.171583+0000 (oldest deadline 2024-03-30T13:41:33.871666+0000) 2024-03-30T13:41:35.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:35 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:35.513+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:35.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:35 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:35.513+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:35.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:35 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:35.476+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:35.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:35 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:35.476+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:35.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:35 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:35.476+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:36.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:36.676+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:50.369582+0000 front 2024-03-30T13:40:50.369573+0000 (oldest deadline 2024-03-30T13:41:21.570032+0000) 2024-03-30T13:41:36.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:36.676+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:01.570529+0000 front 2024-03-30T13:41:01.570573+0000 (oldest deadline 2024-03-30T13:41:25.670739+0000) 2024-03-30T13:41:36.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5[60776]: 2024-03-30T13:41:36.676+0000 7f885316e700 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:09.171626+0000 front 2024-03-30T13:41:09.171583+0000 (oldest deadline 2024-03-30T13:41:33.871666+0000) 2024-03-30T13:41:36.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:36.466+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:36.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:36.466+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:36.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:36.471+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:36.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:36.471+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:36.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:36 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:36.471+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:37.397 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:37 smithi187 podman[80371]: 2024-03-30 13:41:37.101312876 +0000 UTC m=+5.512210270 container stop e70ee05c8ec9dd161c2f9cd82fdb2634ad35838f314db146159e996746e104f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-30T13:41:37.397 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:37 smithi187 podman[80371]: 2024-03-30 13:41:37.130605033 +0000 UTC m=+5.541502438 container died e70ee05c8ec9dd161c2f9cd82fdb2634ad35838f314db146159e996746e104f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:41:37.665 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:37 smithi187 podman[80371]: 2024-03-30 13:41:37.495681496 +0000 UTC m=+5.906578886 container cleanup e70ee05c8ec9dd161c2f9cd82fdb2634ad35838f314db146159e996746e104f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-03-30T13:41:37.665 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:37 smithi187 bash[80371]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5 2024-03-30T13:41:37.665 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:37 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:37.510+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:37.665 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:37 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:37.510+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:37.665 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:37 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:37.510+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:37.666 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:37 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:37.441+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:37.666 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:37 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:37.441+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:37.666 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:37 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:37.441+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:38.022 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:37 smithi187 podman[80401]: 2024-03-30 13:41:37.663706164 +0000 UTC m=+0.556563796 container remove e70ee05c8ec9dd161c2f9cd82fdb2634ad35838f314db146159e996746e104f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-30T13:41:38.359 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:38 smithi187 podman[80495]: 2024-03-30 13:41:37.967282128 +0000 UTC m=+0.017924361 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:41:38.359 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:38 smithi187 podman[80495]: 2024-03-30 13:41:38.072988274 +0000 UTC m=+0.123630531 container create 1e9ee5c173a6659678097f46437f57bf1ac5c79af0f52ca5298ebf7bfc7a856f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-03-30T13:41:38.359 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:38 smithi187 podman[80495]: 2024-03-30 13:41:38.275927282 +0000 UTC m=+0.326569514 container init 1e9ee5c173a6659678097f46437f57bf1ac5c79af0f52ca5298ebf7bfc7a856f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326) 2024-03-30T13:41:38.360 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:38 smithi187 podman[80495]: 2024-03-30 13:41:38.279260908 +0000 UTC m=+0.329903126 container start 1e9ee5c173a6659678097f46437f57bf1ac5c79af0f52ca5298ebf7bfc7a856f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-03-30T13:41:38.360 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:38 smithi187 podman[80495]: 2024-03-30 13:41:38.358632174 +0000 UTC m=+0.409274389 container attach 1e9ee5c173a6659678097f46437f57bf1ac5c79af0f52ca5298ebf7bfc7a856f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True) 2024-03-30T13:41:38.772 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 30 13:41:38 smithi187 podman[80495]: 2024-03-30 13:41:38.573321718 +0000 UTC m=+0.623963944 container died 1e9ee5c173a6659678097f46437f57bf1ac5c79af0f52ca5298ebf7bfc7a856f (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-5-deactivate, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-30T13:41:38.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:38 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:38.402+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:38.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:38 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:38.402+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:38.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:38 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:38.402+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:38.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:38 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:38.509+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:38.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:38 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:38.509+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:38.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:38 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:38.509+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:39.186 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.5.service' 2024-03-30T13:41:39.232 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:41:39.232 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-30T13:41:39.232 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-30T13:41:39.232 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.6 2024-03-30T13:41:39.484 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:39 smithi187 systemd[1]: Stopping Ceph osd.6 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:41:39.484 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:39.376+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:39.485 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:39.376+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:39.485 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:39.376+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:39.771 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:39.482+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:39.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:39.482+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:39.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:39.482+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:39.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:39.531+0000 7f732c78e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:41:39.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:39.531+0000 7f732c78e700 -1 osd.6 155 *** Got signal Terminated *** 2024-03-30T13:41:39.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:39 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:39.531+0000 7f732c78e700 -1 osd.6 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:41:40.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:40 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:40.491+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:40.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:40 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:40.491+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:40.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:40 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:40.491+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:40.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:40 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:40.358+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:40.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:40 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:40.358+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:40.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:40 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:40.358+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:41.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:41 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:41.531+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:41.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:41 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:41.531+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:41.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:41 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:41.531+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:41.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:41 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:41.365+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:41.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:41 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:41.365+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:41.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:41 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:41.365+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:42.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:42 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:42.514+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:42.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:42 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:42.514+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:42.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:42 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:42.514+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:42.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:42 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:42.514+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:18.069310+0000 front 2024-03-30T13:41:18.069282+0000 (oldest deadline 2024-03-30T13:41:42.169253+0000) 2024-03-30T13:41:42.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:42 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:42.317+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:42.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:42 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:42.318+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:42.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:42 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:42.318+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:42.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:42 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:42.318+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:43.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:43 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:43.346+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:43.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:43 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:43.346+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:43.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:43 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:43.346+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:43.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:43 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:43.346+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:43.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:43 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:43.471+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:43.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:43 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:43.471+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:43.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:43 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:43.471+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:43.773 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:43 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:43.471+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:18.069310+0000 front 2024-03-30T13:41:18.069282+0000 (oldest deadline 2024-03-30T13:41:42.169253+0000) 2024-03-30T13:41:44.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:44.447+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:55.266811+0000 front 2024-03-30T13:40:55.266718+0000 (oldest deadline 2024-03-30T13:41:20.566734+0000) 2024-03-30T13:41:44.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:44.447+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:04.067595+0000 front 2024-03-30T13:41:04.067555+0000 (oldest deadline 2024-03-30T13:41:28.167434+0000) 2024-03-30T13:41:44.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:44.447+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:08.668296+0000 front 2024-03-30T13:41:08.668270+0000 (oldest deadline 2024-03-30T13:41:33.968310+0000) 2024-03-30T13:41:44.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6[63066]: 2024-03-30T13:41:44.447+0000 7f7328d82700 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:18.069310+0000 front 2024-03-30T13:41:18.069282+0000 (oldest deadline 2024-03-30T13:41:42.169253+0000) 2024-03-30T13:41:44.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:44.317+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:44.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:44.317+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:44.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:44.317+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:44.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:44 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:44.317+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:45.424 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:45 smithi187 podman[80604]: 2024-03-30 13:41:45.103670098 +0000 UTC m=+5.686442751 container died e4f91e53cf7d2dc3008993b75948c20ca8882c7a87d8df889c7fafbadc9b83ec (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-30T13:41:45.424 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:45 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:45.326+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:45.424 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:45 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:45.326+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:45.424 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:45 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:45.326+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:45.424 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:45 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:45.326+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:45.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:45 smithi187 podman[80604]: 2024-03-30 13:41:45.423470486 +0000 UTC m=+6.006243147 container cleanup e4f91e53cf7d2dc3008993b75948c20ca8882c7a87d8df889c7fafbadc9b83ec (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, ceph=True, org.label-schema.build-date=20240326, RELEASE=squid-4e50bbc, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:41:45.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:45 smithi187 bash[80604]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6 2024-03-30T13:41:45.772 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:45 smithi187 podman[80623]: 2024-03-30 13:41:45.641467396 +0000 UTC m=+0.537381585 container remove e4f91e53cf7d2dc3008993b75948c20ca8882c7a87d8df889c7fafbadc9b83ec (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6, RELEASE=squid-4e50bbc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-30T13:41:46.337 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:46 smithi187 podman[80717]: 2024-03-30 13:41:46.075679565 +0000 UTC m=+0.110273526 container create 65d8e7375e2cb1e97c6212c44d67a678eb77e90fc36331cb0541988b3d1ae978 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, ceph=True, org.label-schema.license=GPLv2) 2024-03-30T13:41:46.337 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:46 smithi187 podman[80717]: 2024-03-30 13:41:45.983107177 +0000 UTC m=+0.017701138 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:41:46.337 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:46 smithi187 podman[80717]: 2024-03-30 13:41:46.304059587 +0000 UTC m=+0.338653553 container init 65d8e7375e2cb1e97c6212c44d67a678eb77e90fc36331cb0541988b3d1ae978 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-03-30T13:41:46.337 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:46 smithi187 podman[80717]: 2024-03-30 13:41:46.307583219 +0000 UTC m=+0.342177181 container start 65d8e7375e2cb1e97c6212c44d67a678eb77e90fc36331cb0541988b3d1ae978 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-4e50bbc, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-30T13:41:46.602 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:46 smithi187 podman[80717]: 2024-03-30 13:41:46.378443621 +0000 UTC m=+0.413037583 container attach 65d8e7375e2cb1e97c6212c44d67a678eb77e90fc36331cb0541988b3d1ae978 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-30T13:41:46.603 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:46 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:46.335+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:46.603 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:46 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:46.335+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:46.603 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:46 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:46.335+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:46.603 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:46 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:46.335+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:46.890 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 30 13:41:46 smithi187 podman[80717]: 2024-03-30 13:41:46.601799396 +0000 UTC m=+0.636393363 container died 65d8e7375e2cb1e97c6212c44d67a678eb77e90fc36331cb0541988b3d1ae978 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-6-deactivate, RELEASE=squid-4e50bbc, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-30T13:41:47.181 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.6.service' 2024-03-30T13:41:47.222 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:41:47.222 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-30T13:41:47.222 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-30T13:41:47.222 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.7 2024-03-30T13:41:47.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:47 smithi187 systemd[1]: Stopping Ceph osd.7 for 9058771a-ee97-11ee-b647-cb9ed24678a4... 2024-03-30T13:41:47.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:47 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:47.370+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:47.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:47 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:47.370+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:47.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:47 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:47.370+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:47.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:47 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:47.370+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:48.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:47 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:47.708+0000 7f226f684700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-30T13:41:48.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:47 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:47.708+0000 7f226f684700 -1 osd.7 155 *** Got signal Terminated *** 2024-03-30T13:41:48.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:47 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:47.708+0000 7f226f684700 -1 osd.7 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-30T13:41:48.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:48 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:48.329+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:48.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:48 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:48.329+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:48.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:48 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:48.329+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:48.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:48 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:48.329+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:49.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:49 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:49.376+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:49.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:49 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:49.376+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:49.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:49 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:49.376+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:49.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:49 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:49.376+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:50.523 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:50 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:50.365+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:50.523 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:50 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:50.365+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:50.523 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:50 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:50.365+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:50.523 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:50 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:50.365+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:51.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:51 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:51.380+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:51.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:51 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:51.380+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:51.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:51 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:51.380+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:51.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:51 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:51.380+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:52.775 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:52 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:52.349+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2024-03-30T13:40:52.350184+0000 front 2024-03-30T13:40:52.350387+0000 (oldest deadline 2024-03-30T13:41:17.650266+0000) 2024-03-30T13:41:52.775 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:52 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:52.349+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6814 osd.1 since back 2024-03-30T13:41:00.952255+0000 front 2024-03-30T13:41:00.952349+0000 (oldest deadline 2024-03-30T13:41:25.652441+0000) 2024-03-30T13:41:52.775 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:52 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:52.349+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6822 osd.2 since back 2024-03-30T13:41:11.854842+0000 front 2024-03-30T13:41:11.854900+0000 (oldest deadline 2024-03-30T13:41:37.154938+0000) 2024-03-30T13:41:52.775 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:52 smithi187 ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7[65321]: 2024-03-30T13:41:52.349+0000 7f226bc78700 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.159:6830 osd.3 since back 2024-03-30T13:41:17.155423+0000 front 2024-03-30T13:41:17.155515+0000 (oldest deadline 2024-03-30T13:41:41.855398+0000) 2024-03-30T13:41:53.477 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:53 smithi187 podman[80821]: 2024-03-30 13:41:53.21637951 +0000 UTC m=+5.776718232 container died b2f37650410f8250af0355f5104156c21bfae155139c61b34d4cf68b856ca102 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-03-30T13:41:53.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:53 smithi187 podman[80821]: 2024-03-30 13:41:53.642896117 +0000 UTC m=+6.203234873 container cleanup b2f37650410f8250af0355f5104156c21bfae155139c61b34d4cf68b856ca102 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-30T13:41:53.772 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:53 smithi187 bash[80821]: ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7 2024-03-30T13:41:54.256 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:53 smithi187 podman[80882]: 2024-03-30 13:41:53.835706687 +0000 UTC m=+0.617743157 container remove b2f37650410f8250af0355f5104156c21bfae155139c61b34d4cf68b856ca102 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_CLEAN=True, RELEASE=squid-4e50bbc, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-03-30T13:41:54.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:54 smithi187 podman[80976]: 2024-03-30 13:41:54.157537272 +0000 UTC m=+0.017796466 image pull b0c4478f1d83795df4d538bf108aee5fac57194348fb16dcdbb3115f3e2fe95e quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d 2024-03-30T13:41:54.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:54 smithi187 podman[80976]: 2024-03-30 13:41:54.261675798 +0000 UTC m=+0.121935027 container create 8fe2476a62bf63e6f38599b2f90cf6dcb766c426aaf20dbaf06de7b5b178b8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326) 2024-03-30T13:41:54.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:54 smithi187 podman[80976]: 2024-03-30 13:41:54.480709542 +0000 UTC m=+0.340968727 container init 8fe2476a62bf63e6f38599b2f90cf6dcb766c426aaf20dbaf06de7b5b178b8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD) 2024-03-30T13:41:54.522 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:54 smithi187 podman[80976]: 2024-03-30 13:41:54.483764441 +0000 UTC m=+0.344023632 container start 8fe2476a62bf63e6f38599b2f90cf6dcb766c426aaf20dbaf06de7b5b178b8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=) 2024-03-30T13:41:55.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:54 smithi187 podman[80976]: 2024-03-30 13:41:54.572390277 +0000 UTC m=+0.432649478 container attach 8fe2476a62bf63e6f38599b2f90cf6dcb766c426aaf20dbaf06de7b5b178b8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-4e50bbc, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-30T13:41:55.022 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 30 13:41:54 smithi187 podman[80976]: 2024-03-30 13:41:54.775868439 +0000 UTC m=+0.636127629 container died 8fe2476a62bf63e6f38599b2f90cf6dcb766c426aaf20dbaf06de7b5b178b8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8f3809717761e681477c3ef1d8d2c03114aee2c597cd7380b93b8c10f47fcb5d, name=ceph-9058771a-ee97-11ee-b647-cb9ed24678a4-osd-7-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-4e50bbc, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-03-30T13:41:55.353 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9058771a-ee97-11ee-b647-cb9ed24678a4@osd.7.service' 2024-03-30T13:41:55.394 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-30T13:41:55.394 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-30T13:41:55.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-03-30T13:44:50.171 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-03-30T13:45:10.973 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-30T13:45:11.009 DEBUG:teuthology.orchestra.run.smithi187:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-30T13:45:11.039 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-30T13:45:11.040 DEBUG:teuthology.misc:Transferring archived files from smithi159:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077/remote/smithi159/crash 2024-03-30T13:45:11.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar c -f - -C /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/crash -- . 2024-03-30T13:45:11.093 INFO:teuthology.orchestra.run.smithi159.stderr:tar: /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-03-30T13:45:11.093 INFO:teuthology.orchestra.run.smithi159.stderr:tar: Error is not recoverable: exiting now 2024-03-30T13:45:11.095 DEBUG:teuthology.misc:Transferring archived files from smithi187:/var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077/remote/smithi187/crash 2024-03-30T13:45:11.095 DEBUG:teuthology.orchestra.run.smithi187:> sudo tar c -f - -C /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/crash -- . 2024-03-30T13:45:11.167 INFO:teuthology.orchestra.run.smithi187.stderr:tar: /var/lib/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-03-30T13:45:11.167 INFO:teuthology.orchestra.run.smithi187.stderr:tar: Error is not recoverable: exiting now 2024-03-30T13:45:11.168 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-30T13:45:11.169 DEBUG:teuthology.orchestra.run.smithi159:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-30T13:45:11.202 INFO:teuthology.orchestra.run.smithi159.stdout:2024-03-30T13:28:02.079163+0000 mon.smithi159 (mon.0) 858 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-30T13:45:11.202 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-30T13:45:11.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo egrep '\[SEC\]' /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-30T13:45:11.273 DEBUG:teuthology.orchestra.run.smithi159:> sudo egrep '\[ERR\]' /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-30T13:45:11.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo egrep '\[WRN\]' /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-30T13:45:11.413 INFO:teuthology.orchestra.run.smithi159.stdout:2024-03-30T13:28:02.079163+0000 mon.smithi159 (mon.0) 858 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-30T13:45:11.413 INFO:tasks.cephadm:Compressing logs... 2024-03-30T13:45:11.413 DEBUG:teuthology.orchestra.run.smithi159:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-30T13:45:11.456 DEBUG:teuthology.orchestra.run.smithi187:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-30T13:45:11.481 INFO:teuthology.orchestra.run.smithi159.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-30T13:45:11.481 INFO:teuthology.orchestra.run.smithi159.stderr:: No such file or directory 2024-03-30T13:45:11.481 INFO:teuthology.orchestra.run.smithi159.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mgr.smithi159.fnasly.log 2024-03-30T13:45:11.482 INFO:teuthology.orchestra.run.smithi159.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-03-30T13:45:11.482 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mgr.smithi159.fnasly.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.1.log 2024-03-30T13:45:11.482 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.3.log 2024-03-30T13:45:11.489 INFO:teuthology.orchestra.run.smithi187.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:: No such file or directory 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.6.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mon.smithi187.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi187.bkvdqd.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mon.smithi187.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.4.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi187.bkvdqd.log: /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.7.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr: 87.8% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi187.bkvdqd.log.gz 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.5.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mgr.smithi187.jpkaud.log 2024-03-30T13:45:11.490 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-03-30T13:45:11.491 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mgr.smithi187.jpkaud.log: 91.6% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-03-30T13:45:11.491 INFO:teuthology.orchestra.run.smithi159.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.0.log 2024-03-30T13:45:11.491 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.1.log: /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi159.jwwwqm.log 2024-03-30T13:45:11.491 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-03-30T13:45:11.491 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi159.jwwwqm.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log 2024-03-30T13:45:11.491 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.cephadm.log: 89.5% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi159.jwwwqm.log.gz 2024-03-30T13:45:11.491 INFO:teuthology.orchestra.run.smithi159.stderr: 85.8% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-03-30T13:45:11.491 INFO:teuthology.orchestra.run.smithi159.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.2.log 2024-03-30T13:45:11.492 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-03-30T13:45:11.492 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.2.log: 91.5% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-03-30T13:45:11.492 INFO:teuthology.orchestra.run.smithi159.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mon.smithi159.log 2024-03-30T13:45:11.492 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-volume.log: 87.6% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-03-30T13:45:11.494 INFO:teuthology.orchestra.run.smithi187.stderr: 90.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-30T13:45:11.494 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log 2024-03-30T13:45:11.494 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-03-30T13:45:11.494 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log: 85.7% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-03-30T13:45:11.496 INFO:teuthology.orchestra.run.smithi187.stderr: 87.6% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-03-30T13:45:11.505 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mon.smithi159.log: 91.3% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-03-30T13:45:11.512 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-volume.log: 91.4% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-03-30T13:45:11.516 INFO:teuthology.orchestra.run.smithi159.stderr: 91.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-30T13:45:11.555 INFO:teuthology.orchestra.run.smithi187.stderr: 89.8% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mgr.smithi187.jpkaud.log.gz 2024-03-30T13:45:11.844 INFO:teuthology.orchestra.run.smithi187.stderr: 92.5% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mon.smithi187.log.gz 2024-03-30T13:45:11.883 INFO:teuthology.orchestra.run.smithi159.stderr: 88.6% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mgr.smithi159.fnasly.log.gz 2024-03-30T13:45:12.867 INFO:teuthology.orchestra.run.smithi159.stderr: 91.9% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-mon.smithi159.log.gz 2024-03-30T13:45:23.608 INFO:teuthology.orchestra.run.smithi159.stderr: 94.5% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.2.log.gz 2024-03-30T13:45:26.341 INFO:teuthology.orchestra.run.smithi187.stderr: 93.9% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.4.log.gz 2024-03-30T13:45:28.730 INFO:teuthology.orchestra.run.smithi187.stderr: 94.2% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.6.log.gz 2024-03-30T13:45:29.229 INFO:teuthology.orchestra.run.smithi159.stderr: 94.1% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.0.log.gz 2024-03-30T13:45:29.945 INFO:teuthology.orchestra.run.smithi159.stderr: 94.2% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.1.log.gz 2024-03-30T13:45:32.177 INFO:teuthology.orchestra.run.smithi187.stderr: 94.1% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.5.log.gz 2024-03-30T13:45:32.454 INFO:teuthology.orchestra.run.smithi159.stderr: 94.1% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.3.log.gz 2024-03-30T13:45:32.456 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-03-30T13:45:32.456 INFO:teuthology.orchestra.run.smithi159.stderr:real 0m20.988s 2024-03-30T13:45:32.456 INFO:teuthology.orchestra.run.smithi159.stderr:user 1m8.252s 2024-03-30T13:45:32.456 INFO:teuthology.orchestra.run.smithi159.stderr:sys 0m2.789s 2024-03-30T13:45:36.591 INFO:teuthology.orchestra.run.smithi187.stderr: 94.0% -- replaced with /var/log/ceph/9058771a-ee97-11ee-b647-cb9ed24678a4/ceph-osd.7.log.gz 2024-03-30T13:45:36.593 INFO:teuthology.orchestra.run.smithi187.stderr: 2024-03-30T13:45:36.593 INFO:teuthology.orchestra.run.smithi187.stderr:real 0m25.123s 2024-03-30T13:45:36.593 INFO:teuthology.orchestra.run.smithi187.stderr:user 1m15.286s 2024-03-30T13:45:36.593 INFO:teuthology.orchestra.run.smithi187.stderr:sys 0m2.958s 2024-03-30T13:45:36.593 INFO:tasks.cephadm:Archiving logs... 2024-03-30T13:45:36.594 DEBUG:teuthology.misc:Transferring archived files from smithi159:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077/remote/smithi159/log 2024-03-30T13:45:36.595 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-30T13:45:46.453 DEBUG:teuthology.misc:Transferring archived files from smithi187:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077/remote/smithi187/log 2024-03-30T13:45:46.454 DEBUG:teuthology.orchestra.run.smithi187:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-30T13:45:57.345 INFO:tasks.cephadm:Removing cluster... 2024-03-30T13:45:57.346 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 --force 2024-03-30T13:45:57.723 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9058771a-ee97-11ee-b647-cb9ed24678a4 --force 2024-03-30T13:45:58.101 INFO:tasks.cephadm:Removing cephadm ... 2024-03-30T13:45:58.102 DEBUG:teuthology.orchestra.run.smithi159:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-30T13:45:58.117 DEBUG:teuthology.orchestra.run.smithi187:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-30T13:45:58.132 INFO:tasks.cephadm:Teardown complete 2024-03-30T13:45:58.132 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-30T13:45:58.143 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-30T13:45:58.144 DEBUG:teuthology.orchestra.run.smithi159:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-30T13:45:58.160 DEBUG:teuthology.orchestra.run.smithi187:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-30T13:45:58.171 INFO:teuthology.orchestra.run.smithi159.stderr:bash: line 1: ntpq: command not found 2024-03-30T13:45:58.176 INFO:teuthology.orchestra.run.smithi159.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-30T13:45:58.176 INFO:teuthology.orchestra.run.smithi159.stdout:=============================================================================== 2024-03-30T13:45:58.177 INFO:teuthology.orchestra.run.smithi159.stdout:^+ hv01.front.sepia.ceph.com 2 8 377 211 +100us[ +101us] +/- 84ms 2024-03-30T13:45:58.177 INFO:teuthology.orchestra.run.smithi159.stdout:^* hv02.front.sepia.ceph.com 3 8 377 84 -299us[ -297us] +/- 78ms 2024-03-30T13:45:58.177 INFO:teuthology.orchestra.run.smithi159.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 19 +285us[ +285us] +/- 74ms 2024-03-30T13:45:58.177 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:45:58.185 INFO:teuthology.orchestra.run.smithi187.stderr:bash: line 1: ntpq: command not found 2024-03-30T13:45:58.189 INFO:teuthology.orchestra.run.smithi187.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-30T13:45:58.190 INFO:teuthology.orchestra.run.smithi187.stdout:=============================================================================== 2024-03-30T13:45:58.190 INFO:teuthology.orchestra.run.smithi187.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 19 +56us[ +75us] +/- 87ms 2024-03-30T13:45:58.190 INFO:teuthology.orchestra.run.smithi187.stdout:^* hv02.front.sepia.ceph.com 3 6 377 17 -324us[ -305us] +/- 79ms 2024-03-30T13:45:58.190 INFO:teuthology.orchestra.run.smithi187.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 17 +273us[ +292us] +/- 74ms 2024-03-30T13:45:58.190 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-30T13:45:58.191 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-30T13:45:58.201 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-30T13:45:58.203 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-30T13:45:58.212 DEBUG:teuthology.orchestra.run.smithi159:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-30T13:45:58.303 DEBUG:teuthology.orchestra.run.smithi187:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-30T13:45:58.382 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-30T13:45:58.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-30T13:45:58.414 DEBUG:teuthology.orchestra.run.smithi187:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-30T13:45:58.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-30T13:45:58.445 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-30T13:45:58.457 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-30T13:45:58.467 INFO:teuthology.task.internal:Duration was 2191.893296 seconds 2024-03-30T13:45:58.467 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-30T13:45:58.477 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-30T13:45:58.477 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-30T13:45:58.479 DEBUG:teuthology.orchestra.run.smithi187:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-30T13:45:58.521 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-30T13:45:58.527 INFO:teuthology.orchestra.run.smithi187.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-30T13:45:58.810 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-30T13:45:58.811 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi159.front.sepia.ceph.com 2024-03-30T13:45:58.811 DEBUG:teuthology.orchestra.run.smithi159:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-30T13:45:58.867 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi187.front.sepia.ceph.com 2024-03-30T13:45:58.868 DEBUG:teuthology.orchestra.run.smithi187:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-30T13:45:58.890 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-30T13:45:58.890 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-30T13:45:58.910 DEBUG:teuthology.orchestra.run.smithi187:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-30T13:45:58.959 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-30T13:45:58.959 DEBUG:teuthology.orchestra.run.smithi159:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-30T13:45:58.979 DEBUG:teuthology.orchestra.run.smithi187:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-30T13:46:12.278 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-30T13:46:12.290 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-30T13:46:12.290 DEBUG:teuthology.orchestra.run.smithi159:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-30T13:46:12.318 DEBUG:teuthology.orchestra.run.smithi187:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-30T13:46:12.351 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-30T13:46:12.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-30T13:46:12.365 DEBUG:teuthology.orchestra.run.smithi187:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-30T13:46:12.388 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = core 2024-03-30T13:46:12.420 INFO:teuthology.orchestra.run.smithi187.stdout:kernel.core_pattern = core 2024-03-30T13:46:12.462 DEBUG:teuthology.orchestra.run.smithi159:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-30T13:46:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-30T13:46:12.476 DEBUG:teuthology.orchestra.run.smithi187:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-30T13:46:12.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-30T13:46:12.517 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-30T13:46:12.528 INFO:teuthology.task.internal:Transferring archived files... 2024-03-30T13:46:12.529 DEBUG:teuthology.misc:Transferring archived files from smithi159:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077/remote/smithi159 2024-03-30T13:46:12.529 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-30T13:46:12.597 DEBUG:teuthology.misc:Transferring archived files from smithi187:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077/remote/smithi187 2024-03-30T13:46:12.597 DEBUG:teuthology.orchestra.run.smithi187:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-30T13:46:12.644 INFO:teuthology.task.internal:Removing archive directory... 2024-03-30T13:46:12.644 DEBUG:teuthology.orchestra.run.smithi159:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-30T13:46:12.646 DEBUG:teuthology.orchestra.run.smithi187:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-30T13:46:12.693 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-30T13:46:12.705 INFO:teuthology.task.internal:Not uploading archives. 2024-03-30T13:46:12.705 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-30T13:46:12.715 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-30T13:46:12.716 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-30T13:46:12.718 DEBUG:teuthology.orchestra.run.smithi187:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-30T13:46:12.730 INFO:teuthology.orchestra.run.smithi159.stdout: 265566 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 30 13:46 /home/ubuntu/cephtest 2024-03-30T13:46:12.748 INFO:teuthology.orchestra.run.smithi187.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 30 13:46 /home/ubuntu/cephtest 2024-03-30T13:46:12.749 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-30T13:46:12.760 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-30T13:46:12.785 INFO:teuthology.nuke:Checking targets against current locks 2024-03-30T13:46:12.815 DEBUG:teuthology.nuke:shortname: smithi159 2024-03-30T13:46:12.815 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-30T13:46:12.839 DEBUG:teuthology.nuke:shortname: smithi187 2024-03-30T13:46:12.839 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-30T13:46:12.843 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi159.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-30 13:01:13.794067', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAhS3uAOwmqwZkto2LEa9WB2bREDcFW5GXpupBhN9iSSKjQfNgtY6ZWb94PW8qsfSftV1I1158op8sF2x5hjh6U='} 2024-03-30T13:46:12.867 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi187.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_21:08:02-orch-squid-distro-default-smithi/7630077', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-30 13:01:13.795325', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPGpbOiGjH1fjHnJig6Re17NRUTfgotqtnTfySHE65gHF2Tjvv8k8orLSTil90JtgqeJi+2x8FKG4kseo4KP9Fg='} 2024-03-30T13:46:12.892 INFO:teuthology.orchestra.console.smithi159:Power off 2024-03-30T13:46:12.892 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-30T13:46:12.919 DEBUG:teuthology.orchestra.console.smithi159:power off output: Chassis Power Control: Down/Off 2024-03-30T13:46:12.919 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-30T13:46:12.934 INFO:teuthology.orchestra.console.smithi187:Power off 2024-03-30T13:46:12.935 DEBUG:teuthology.orchestra.console.smithi187:pexpect command: ipmitool -H smithi187.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-30T13:46:12.945 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is on 2024-03-30T13:46:12.961 DEBUG:teuthology.orchestra.console.smithi187:power off output: Chassis Power Control: Down/Off 2024-03-30T13:46:12.961 DEBUG:teuthology.orchestra.console.smithi187:pexpect command: ipmitool -H smithi187.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-30T13:46:12.986 DEBUG:teuthology.orchestra.console.smithi187:check power output: Chassis Power is on 2024-03-30T13:46:16.946 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-30T13:46:16.986 DEBUG:teuthology.orchestra.console.smithi187:pexpect command: ipmitool -H smithi187.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-30T13:46:17.056 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is on 2024-03-30T13:46:17.095 DEBUG:teuthology.orchestra.console.smithi187:check power output: Chassis Power is on 2024-03-30T13:46:21.058 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-30T13:46:21.097 DEBUG:teuthology.orchestra.console.smithi187:pexpect command: ipmitool -H smithi187.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-30T13:46:21.169 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is on 2024-03-30T13:46:21.207 DEBUG:teuthology.orchestra.console.smithi187:check power output: Chassis Power is off 2024-03-30T13:46:21.308 INFO:teuthology.orchestra.console.smithi187:Power off completed 2024-03-30T13:46:25.170 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-30T13:46:25.279 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is off 2024-03-30T13:46:25.380 INFO:teuthology.orchestra.console.smithi159:Power off completed 2024-03-30T13:46:25.484 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2191.8932960033417 failure_reason: '"2024-03-30T13:28:02.079163+0000 mon.smithi159 (mon.0) 858 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-03-30T13:46:25.484 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-30T13:46:25.583 INFO:teuthology.run:FAIL